From c0595c21c53c74557ebfa3799770c002c391ffdc Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 3 Mar 2016 11:05:37 -0800 Subject: [PATCH] Halt stack pops at all error states, not just error trees --- spec/runtime/stack_spec.cc | 113 ++++++++++++++++++++----------------- src/runtime/parser.c | 90 +++++++++++++++-------------- src/runtime/stack.c | 15 ++--- src/runtime/stack.h | 12 +++- 4 files changed, 127 insertions(+), 103 deletions(-) diff --git a/spec/runtime/stack_spec.cc b/spec/runtime/stack_spec.cc index e65c61da..bd17bda9 100644 --- a/spec/runtime/stack_spec.cc +++ b/spec/runtime/stack_spec.cc @@ -132,61 +132,68 @@ describe("Stack", [&]() { it("removes the given number of nodes from the stack", [&]() { // . <──0── A* - StackSliceArray slices = ts_stack_pop(stack, 0, 2, false); - AssertThat(slices.size, Equals(1)); + StackPopResult pop_result = ts_stack_pop(stack, 0, 2, false); + AssertThat(pop_result.status, Equals(StackPopResult::StackPopSucceeded)); + AssertThat(pop_result.slices.size, Equals(1)); - StackSlice slice = slices.contents[0]; + StackSlice slice = pop_result.slices.contents[0]; AssertThat(slice.trees, Equals(vector({ trees[1], trees[2] }))); AssertThat(*ts_stack_head(stack, 0), Equals({stateA, tree_len})); - free_slice_array(&slices); + free_slice_array(&pop_result.slices); // .* - slices = ts_stack_pop(stack, 0, 1, false); - AssertThat(slices.size, Equals(1)); + pop_result = ts_stack_pop(stack, 0, 1, false); + AssertThat(pop_result.status, Equals(StackPopResult::StackPopSucceeded)); + AssertThat(pop_result.slices.size, Equals(1)); - slice = slices.contents[0]; + slice = pop_result.slices.contents[0]; AssertThat(slice.trees, Equals(vector({ trees[0] }))); AssertThat(ts_stack_head(stack, 0), Equals(nullptr)); - free_slice_array(&slices); + free_slice_array(&pop_result.slices); }); it("does not count 'extra' trees toward the count", [&]() { trees[1]->extra = true; // .* - StackSliceArray slices = ts_stack_pop(stack, 0, 2, false); - AssertThat(slices.size, Equals(1)); + StackPopResult pop_result = ts_stack_pop(stack, 0, 2, false); + AssertThat(pop_result.status, Equals(StackPopResult::StackPopSucceeded)); + AssertThat(pop_result.slices.size, Equals(1)); - StackSlice slice = slices.contents[0]; + StackSlice slice = pop_result.slices.contents[0]; AssertThat(slice.trees, Equals(vector({ trees[0], trees[1], trees[2] }))); AssertThat(ts_stack_head(stack, 0), Equals(nullptr)); - free_slice_array(&slices); + free_slice_array(&pop_result.slices); }); it("pops the entire stack when given a negative count", [&]() { // .* - StackSliceArray slices = ts_stack_pop(stack, 0, -1, false); - AssertThat(slices.size, Equals(1)); + StackPopResult pop_result = ts_stack_pop(stack, 0, -1, false); + AssertThat(pop_result.status, Equals(StackPopResult::StackPopSucceeded)); + AssertThat(pop_result.slices.size, Equals(1)); - StackSlice slice = slices.contents[0]; + StackSlice slice = pop_result.slices.contents[0]; AssertThat(slice.trees, Equals(vector({ trees[0], trees[1], trees[2] }))); - free_slice_array(&slices); + free_slice_array(&pop_result.slices); }); - it("stops immediately when removing an error", [&]() { - trees[2]->symbol = ts_builtin_sym_error; + it("stops immediately after popping a node with the error state", [&]() { + // . <──0── A <──1── B <──2── C <──3── ? <──4── D* + ts_stack_push(stack, 0, trees[3], ts_parse_state_error); + ts_stack_push(stack, 0, trees[4], stateD); - StackSliceArray slices = ts_stack_pop(stack, 0, 2, false); - AssertThat(slices.size, Equals(1)); + StackPopResult pop_result = ts_stack_pop(stack, 0, 3, false); + AssertThat(pop_result.status, Equals(StackPopResult::StackPopStoppedAtError)); + AssertThat(pop_result.slices.size, Equals(1)); - StackSlice slice = slices.contents[0]; - AssertThat(slice.trees, Equals(vector({ trees[2] }))); - AssertThat(ts_stack_top_state(stack, 0), Equals(stateB)); + StackSlice slice = pop_result.slices.contents[0]; + AssertThat(slice.trees, Equals(vector({ trees[3], trees[4] }))); + AssertThat(ts_stack_top_state(stack, 0), Equals(stateC)); - free_slice_array(&slices); + free_slice_array(&pop_result.slices); }); }); @@ -209,15 +216,15 @@ describe("Stack", [&]() { // ↑ // └─* ts_stack_push(stack, 0, trees[3], stateD); - StackSliceArray slices = ts_stack_pop(stack, 1, 1, false); + StackPopResult pop_result = ts_stack_pop(stack, 1, 1, false); AssertThat(ts_stack_head_count(stack), Equals(2)); AssertThat(*ts_stack_head(stack, 0), Equals({stateD, tree_len * 4})); AssertThat(*ts_stack_head(stack, 1), Equals({stateB, tree_len * 2})); - AssertThat(slices.size, Equals(1)); - StackSlice slice = slices.contents[0]; + AssertThat(pop_result.slices.size, Equals(1)); + StackSlice slice = pop_result.slices.contents[0]; AssertThat(slice.trees.size, Equals(1)); - free_slice_array(&slices); + free_slice_array(&pop_result.slices); // . <──0── A <──1── B <──2── C <──3── D* // ↑ @@ -342,15 +349,15 @@ describe("Stack", [&]() { // . <──0── A <──1── B <──2── C* // ↑ // └───5─── F* - StackSliceArray slices = ts_stack_pop(stack, 0, 2, false); + StackPopResult pop_result = ts_stack_pop(stack, 0, 2, false); - AssertThat(slices.size, Equals(2)); - StackSlice slice1 = slices.contents[0]; + AssertThat(pop_result.slices.size, Equals(2)); + StackSlice slice1 = pop_result.slices.contents[0]; AssertThat(slice1.head_index, Equals(0)); AssertThat(ts_stack_top_state(stack, 0), Equals(stateC)); AssertThat(slice1.trees, Equals(vector({ trees[3], trees[4] }))); - StackSlice slice2 = slices.contents[1]; + StackSlice slice2 = pop_result.slices.contents[1]; AssertThat(slice2.head_index, Equals(1)); AssertThat(ts_stack_top_state(stack, 1), Equals(stateF)); AssertThat(slice2.trees, Equals(vector({ trees[6], trees[7] }))); @@ -359,7 +366,7 @@ describe("Stack", [&]() { AssertThat(*ts_stack_head(stack, 0), Equals({stateC, tree_len * 3})); AssertThat(*ts_stack_head(stack, 1), Equals({stateF, tree_len * 3})); - free_slice_array(&slices); + free_slice_array(&pop_result.slices); }); }); @@ -375,16 +382,16 @@ describe("Stack", [&]() { // . <──0── A <──1── B <──2── C <──3── D <──4── E* // ↑ | // └───5─── F <──6── G <──7───┘ - StackSliceArray slices = ts_stack_pop(stack, 0, 1, false); - AssertThat(slices.size, Equals(1)); - StackSlice slice1 = slices.contents[0]; + StackPopResult pop_result = ts_stack_pop(stack, 0, 1, false); + AssertThat(pop_result.slices.size, Equals(1)); + StackSlice slice1 = pop_result.slices.contents[0]; AssertThat(slice1.head_index, Equals(0)); AssertThat(slice1.trees, Equals(vector({ trees[8] }))); AssertThat(ts_stack_head_count(stack), Equals(1)); AssertThat(ts_stack_top_state(stack, 0), Equals(stateE)); - free_slice_array(&slices); + free_slice_array(&pop_result.slices); }); }); @@ -393,17 +400,17 @@ describe("Stack", [&]() { it("returns one entry for that head, with the first path of trees", [&]() { tree_selection_spy.tree_to_return = trees[2]; - // . <──0── A <──1── B* - StackSliceArray slices = ts_stack_pop(stack, 0, 3, false); + // . <──0── A <──1── B* + StackPopResult pop_result = ts_stack_pop(stack, 0, 3, false); AssertThat(ts_stack_head_count(stack), Equals(1)); AssertThat(*ts_stack_head(stack, 0), Equals({stateB, tree_len * 2})); - AssertThat(slices.size, Equals(1)); - StackSlice slice1 = slices.contents[0]; + AssertThat(pop_result.slices.size, Equals(1)); + StackSlice slice1 = pop_result.slices.contents[0]; AssertThat(slice1.head_index, Equals(0)); AssertThat(slice1.trees, Equals(vector({ trees[2], trees[3], trees[4] }))); - free_slice_array(&slices); + free_slice_array(&pop_result.slices); }); }); @@ -411,17 +418,17 @@ describe("Stack", [&]() { it("returns one entry for that head, with the second path of trees", [&]() { tree_selection_spy.tree_to_return = trees[4]; - // . <──0── A <──1── B* - StackSliceArray slices = ts_stack_pop(stack, 0, 3, false); + // . <──0── A <──1── B* + StackPopResult pop_result = ts_stack_pop(stack, 0, 3, false); AssertThat(ts_stack_head_count(stack), Equals(1)); AssertThat(*ts_stack_head(stack, 0), Equals({stateB, tree_len * 2})); - AssertThat(slices.size, Equals(1)); - StackSlice slice1 = slices.contents[0]; + AssertThat(pop_result.slices.size, Equals(1)); + StackSlice slice1 = pop_result.slices.contents[0]; AssertThat(slice1.head_index, Equals(0)); AssertThat(slice1.trees, Equals(vector({ trees[5], trees[6], trees[7] }))) - free_slice_array(&slices); + free_slice_array(&pop_result.slices); }); }); }); @@ -460,27 +467,27 @@ describe("Stack", [&]() { // ├───4─── E <──5── F* // | // └───7─── G <──8── H* - StackSliceArray slices = ts_stack_pop(stack, 0, 2, false); + StackPopResult pop_result = ts_stack_pop(stack, 0, 2, false); AssertThat(ts_stack_head_count(stack), Equals(3)); - AssertThat(slices.size, Equals(3)); + AssertThat(pop_result.slices.size, Equals(3)); - StackSlice slice1 = slices.contents[0]; + StackSlice slice1 = pop_result.slices.contents[0]; AssertThat(ts_stack_top_state(stack, 0), Equals(stateC)); AssertThat(slice1.head_index, Equals(0)); AssertThat(slice1.trees, Equals(vector({ trees[3], trees[10] }))) - StackSlice slice2 = slices.contents[1]; + StackSlice slice2 = pop_result.slices.contents[1]; AssertThat(ts_stack_top_state(stack, 1), Equals(stateF)); AssertThat(slice2.head_index, Equals(1)); AssertThat(slice2.trees, Equals(vector({ trees[6], trees[10] }))) - StackSlice slice3 = slices.contents[2]; + StackSlice slice3 = pop_result.slices.contents[2]; AssertThat(ts_stack_top_state(stack, 2), Equals(stateH)); AssertThat(slice3.head_index, Equals(2)); AssertThat(slice3.trees, Equals(vector({ trees[9], trees[10] }))) - free_slice_array(&slices); + free_slice_array(&pop_result.slices); }); }); }); diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 72cb000e..84459614 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -52,7 +52,8 @@ struct LookaheadState { typedef struct { TSSymbol symbol; TSStateId next_state; - size_t depth; + size_t stack_depth; + size_t child_count; size_t skipped_subtree_count; size_t in_progress_state_count; } ErrorRepair; @@ -75,15 +76,16 @@ typedef enum { */ static StackSlice ts_parser__pop_one(TSParser *self, int head_index, int count) { - StackSliceArray slices = ts_stack_pop(self->stack, head_index, count, true); - assert(slices.size > 0); - assert(slices.contents[0].head_index == head_index); - for (size_t i = 1; i < slices.size; i++) { - ts_tree_array_clear(&slices.contents[i].trees); - array_delete(&slices.contents[i].trees); - ts_stack_remove_head(self->stack, slices.contents[i].head_index); + StackPopResult pop = ts_stack_pop(self->stack, head_index, count, true); + assert(pop.status == StackPopSucceeded); + assert(pop.slices.size > 0); + assert(pop.slices.contents[0].head_index == head_index); + for (size_t i = 1; i < pop.slices.size; i++) { + ts_tree_array_clear(&pop.slices.contents[i].trees); + array_delete(&pop.slices.contents[i].trees); + ts_stack_remove_head(self->stack, pop.slices.contents[i].head_index); } - return slices.contents[0]; + return pop.slices.contents[0]; } static ParseActionResult ts_parser__breakdown_top_of_stack(TSParser *self, @@ -91,18 +93,18 @@ static ParseActionResult ts_parser__breakdown_top_of_stack(TSParser *self, TSTree *last_child = NULL; do { - StackSliceArray slices = ts_stack_pop(self->stack, head, 1, false); - if (!slices.size) + StackPopResult pop = ts_stack_pop(self->stack, head, 1, false); + if (!pop.slices.size) return ParseActionFailed; - assert(slices.size > 0); + assert(pop.slices.size > 0); /* * Since only one entry (not counting extra trees) is being popped from the * stack, there should only be one possible array of removed trees. */ - for (size_t i = 0; i < slices.size; i++) { - StackSlice slice = slices.contents[i]; + for (size_t i = 0; i < pop.slices.size; i++) { + StackSlice slice = pop.slices.contents[i]; TreeArray removed_trees = slice.trees; TSTree *parent = *array_front(&removed_trees); int head_index = slice.head_index; @@ -349,28 +351,26 @@ static ReduceResult ts_parser__reduce(TSParser *self, int head, TSSymbol symbol, int child_count, bool extra, bool fragile) { array_clear(&self->reduce_parents); TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, symbol); - StackSliceArray slices = - ts_stack_pop(self->stack, head, child_count, false); - if (!slices.size) + StackPopResult pop = ts_stack_pop(self->stack, head, child_count, false); + if (!pop.slices.size) return ReduceFailed; - if (slices.contents[0].trees.size && - slices.contents[0].trees.contents[0]->symbol == ts_builtin_sym_error) { + if (pop.status == StackPopStoppedAtError) { if (self->partial_pop.size) { ts_tree_array_clear(&self->partial_pop); array_delete(&self->partial_pop); } - self->partial_pop = slices.contents[0].trees; - for (size_t i = 1; i < slices.size; i++) { - ts_tree_array_clear(&slices.contents[i].trees); - array_delete(&slices.contents[i].trees); + self->partial_pop = pop.slices.contents[0].trees; + for (size_t i = 1; i < pop.slices.size; i++) { + ts_tree_array_clear(&pop.slices.contents[i].trees); + array_delete(&pop.slices.contents[i].trees); } return ReduceStoppedAtError; } size_t removed_heads = 0; - for (size_t i = 0; i < slices.size; i++) { - StackSlice slice = slices.contents[i]; + for (size_t i = 0; i < pop.slices.size; i++) { + StackSlice slice = pop.slices.contents[i]; /* * If the same set of trees led to a previous stack head, reuse the parent @@ -492,7 +492,7 @@ static ReduceResult ts_parser__reduce(TSParser *self, int head, TSSymbol symbol, ts_tree_release(*parent); } - if (removed_heads < slices.size) + if (removed_heads < pop.slices.size) return ReduceSucceeded; else return ReduceMerged; @@ -555,8 +555,11 @@ static ParseActionResult ts_parser__repair_error(TSParser *self, int head_index, const TSParseAction *actions, size_t action_count) { StackEntry *entry_below = ts_stack_head(self->stack, head_index); - while (entry_below && entry_below->state == ts_parse_state_error) + size_t initial_depth = 0; + while (entry_below && entry_below->state == ts_parse_state_error) { entry_below = ts_stack_entry_next(entry_below, 0); + initial_depth++; + } bool has_repair = false; ErrorRepair best_repair; @@ -575,7 +578,7 @@ static ParseActionResult ts_parser__repair_error(TSParser *self, int head_index, continue; StackEntry *entry = entry_below; - size_t depth = 0; + size_t depth = initial_depth; size_t child_count = 0; size_t in_progress_state_count = 0; @@ -599,16 +602,12 @@ static ParseActionResult ts_parser__repair_error(TSParser *self, int head_index, ErrorRepair repair = { .symbol = action.data.symbol, .next_state = next_state, - .depth = depth, + .child_count = action.data.child_count, + .stack_depth = depth, .skipped_subtree_count = child_count + count_above_error - action.data.child_count, .in_progress_state_count = in_progress_state_count, }; - LOG_ACTION("REPAIR_GOOD sym:%s, in_progress:%lu, skipped:%lu!", - SYM_NAME(action.data.symbol), - in_progress_state_count, - repair.skipped_subtree_count); - if (!has_repair || ts_parser__error_repair_is_better(best_repair, repair)) { has_repair = true; @@ -637,6 +636,15 @@ static ParseActionResult ts_parser__repair_error(TSParser *self, int head_index, return ParseActionRemoved; } + LOG_ACTION( + "repair_found sym:%s, child_count:%lu, skip_count:%lu, match_count:%lu, DEPTH:%lu", + SYM_NAME(best_repair.symbol), + best_repair.child_count, + best_repair.in_progress_state_count, + best_repair.skipped_subtree_count, + best_repair.stack_depth + ); + // Pop any trees that were skipped. Make a new extra error node that contains // them and the error leaf node. StackSlice slice = ts_parser__pop_one(self, head_index, best_repair.skipped_subtree_count); @@ -650,7 +658,7 @@ static ParseActionResult ts_parser__repair_error(TSParser *self, int head_index, // Pop any additional trees that are needed for the chosen reduce action. Make // a new wrapper node of the chosen symbol that contains them, the error node, // and the trees that were popped above the error node. - slice = ts_parser__pop_one(self, head_index, best_repair.depth - (error->child_count - 1)); + slice = ts_parser__pop_one(self, head_index, best_repair.stack_depth - (error->child_count - 1)); array_push(&slice.trees, error); for (size_t i = 1; i < self->partial_pop.size; i++) array_push(&slice.trees, self->partial_pop.contents[i]); @@ -698,12 +706,12 @@ static ParseActionResult ts_parser__start(TSParser *self, TSInput input, } static ParseActionResult ts_parser__accept(TSParser *self, int head) { - StackSliceArray slices = ts_stack_pop(self->stack, head, -1, true); - if (!slices.size) + StackPopResult pop = ts_stack_pop(self->stack, head, -1, true); + if (!pop.slices.size) goto error; - for (size_t j = 0; j < slices.size; j++) { - StackSlice slice = slices.contents[j]; + for (size_t j = 0; j < pop.slices.size; j++) { + StackSlice slice = pop.slices.contents[j]; TreeArray trees = slice.trees; for (size_t i = trees.size - 1; i + 1 > 0; i--) { @@ -732,8 +740,8 @@ static ParseActionResult ts_parser__accept(TSParser *self, int head) { return ParseActionRemoved; error: - if (slices.size) { - StackSlice slice = *array_front(&slices); + if (pop.slices.size) { + StackSlice slice = *array_front(&pop.slices); for (size_t i = 0; i < slice.trees.size; i++) ts_tree_release(slice.trees.contents[i]); array_delete(&slice.trees); diff --git a/src/runtime/stack.c b/src/runtime/stack.c index 0aac66cf..909daecd 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -304,7 +304,7 @@ int ts_stack_split(Stack *self, int head_index) { return ts_stack__add_head(self, head); } -StackSliceArray ts_stack_pop(Stack *self, int head_index, int child_count, +StackPopResult ts_stack_pop(Stack *self, int head_index, int child_count, bool count_extra) { array_clear(&self->slices); array_clear(&self->pop_paths); @@ -327,6 +327,7 @@ StackSliceArray ts_stack_pop(Stack *self, int head_index, int child_count, * of child trees have been collected along every path. */ bool all_paths_done = false; + int status = StackPopSucceeded; while (!all_paths_done) { all_paths_done = true; @@ -362,14 +363,16 @@ StackSliceArray ts_stack_pop(Stack *self, int head_index, int child_count, next_path->is_shared = false; } - if (successor.tree->extra && !count_extra) + if (!count_extra && successor.tree->extra) next_path->goal_tree_count++; + ts_tree_retain(successor.tree); if (!array_push(&next_path->trees, successor.tree)) goto error; next_path->node = successor.node; - if (successor.tree->symbol == ts_builtin_sym_error && !count_extra) { + if (!count_extra && node->entry.state == ts_parse_state_error) { + status = StackPopStoppedAtError; next_path->goal_tree_count = next_path->trees.size; } } @@ -416,13 +419,11 @@ StackSliceArray ts_stack_pop(Stack *self, int head_index, int child_count, } stack_node_release(previous_head, &self->node_pool); - return self->slices; + return (StackPopResult){ .status = status, .slices = self->slices }; error: array_delete(&initial_path.trees); - StackSliceArray slices; - array_init(&slices); - return slices; + return (StackPopResult){StackPopFailed, self->slices}; } void ts_stack_shrink(Stack *self, int head_index, int count) { diff --git a/src/runtime/stack.h b/src/runtime/stack.h index ee0ec67a..34b82657 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -29,6 +29,15 @@ typedef enum { StackPushContinued, } StackPushResult; +typedef struct { + enum { + StackPopFailed, + StackPopStoppedAtError, + StackPopSucceeded, + } status; + StackSliceArray slices; +} StackPopResult; + typedef int (*TreeSelectionFunction)(void *, TSTree *, TSTree *); /* @@ -85,8 +94,7 @@ StackPushResult ts_stack_push(Stack *, int head_index, TSTree *, TSStateId); * which had previously been merged. It returns a struct that indicates the * index of each revealed head and the trees removed from that head. */ -StackSliceArray ts_stack_pop(Stack *, int head_index, int count, - bool count_extra); +StackPopResult ts_stack_pop(Stack *, int head_index, int count, bool count_extra); /* * Remove the given number of entries from the given head of the stack.