From 6bce6da1e6f334afd71b7d79b53787c98a79c97a Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 31 Mar 2016 12:03:07 -0700 Subject: [PATCH] Store `verifying` flag within parse stack --- spec/runtime/stack_spec.cc | 143 ++++++++++++++++++++++++------------- src/runtime/parser.c | 100 +++++++++++++------------- src/runtime/stack.c | 62 ++++++++++++---- src/runtime/stack.h | 8 ++- 4 files changed, 197 insertions(+), 116 deletions(-) diff --git a/spec/runtime/stack_spec.cc b/spec/runtime/stack_spec.cc index 1e768bcb..fe11acd3 100644 --- a/spec/runtime/stack_spec.cc +++ b/spec/runtime/stack_spec.cc @@ -19,6 +19,10 @@ enum { }; struct TreeSelectionSpy { + TreeSelectionSpy() : + call_count(0), + tree_to_return(nullptr), + arguments{nullptr, nullptr} {} int call_count; TSTree *tree_to_return; const TSTree *arguments[2]; @@ -71,7 +75,7 @@ vector get_stack_entries(Stack *stack, int head_index) { ts_stack_pop_until( stack, head_index, - [](void *payload, TSStateId state, size_t tree_count, bool is_done) { + [](void *payload, TSStateId state, size_t tree_count, bool is_done, bool is_pending) { auto entries = static_cast *>(payload); StackEntry entry = {state, tree_count}; if (find(entries->begin(), entries->end(), entry) == entries->end()) @@ -87,7 +91,7 @@ describe("Stack", [&]() { Stack *stack; const size_t tree_count = 11; TSTree *trees[tree_count]; - TreeSelectionSpy tree_selection_spy{0, NULL, {NULL, NULL}}; + TreeSelectionSpy tree_selection_spy; TSLength tree_len = {2, 3, 0, 3}; TSSymbolMetadata metadata = {true, true, true, true}; @@ -123,17 +127,17 @@ describe("Stack", [&]() { AssertThat(ts_stack_top_position(stack, 0), Equals(ts_length_zero())); // . <──0── A* - ts_stack_push(stack, 0, trees[0], stateA); + ts_stack_push(stack, 0, trees[0], false, stateA); AssertThat(ts_stack_top_state(stack, 0), Equals(stateA)); AssertThat(ts_stack_top_position(stack, 0), Equals(tree_len)); // . <──0── A <──1── B* - ts_stack_push(stack, 0, trees[1], stateB); + ts_stack_push(stack, 0, trees[1], false, stateB); AssertThat(ts_stack_top_state(stack, 0), Equals(stateB)); AssertThat(ts_stack_top_position(stack, 0), Equals(tree_len * 2)); // . <──0── A <──1── B <──2── C* - ts_stack_push(stack, 0, trees[2], stateC); + ts_stack_push(stack, 0, trees[2], false, stateC); AssertThat(ts_stack_top_state(stack, 0), Equals(stateC)); AssertThat(ts_stack_top_position(stack, 0), Equals(tree_len * 3)); @@ -149,9 +153,9 @@ describe("Stack", [&]() { describe("popping nodes from the stack", [&]() { before_each([&]() { // . <──0── A <──1── B <──2── C* - ts_stack_push(stack, 0, trees[0], stateA); - ts_stack_push(stack, 0, trees[1], stateB); - ts_stack_push(stack, 0, trees[2], stateC); + ts_stack_push(stack, 0, trees[0], false, stateA); + ts_stack_push(stack, 0, trees[1], false, stateB); + ts_stack_push(stack, 0, trees[2], false, stateC); }); it("removes the given number of nodes from the stack", [&]() { @@ -207,8 +211,8 @@ describe("Stack", [&]() { describe("when an error state exists above the given depth", [&]() { it("stops popping nodes at the error", [&]() { // . <──0── A <──1── B <──2── C <──3── ERROR <──4── D* - ts_stack_push(stack, 0, trees[3], ts_parse_state_error); - ts_stack_push(stack, 0, trees[4], stateD); + ts_stack_push(stack, 0, trees[3], false, ts_parse_state_error); + ts_stack_push(stack, 0, trees[4], false, stateD); StackPopResult pop_result = ts_stack_pop_count(stack, 0, 3); AssertThat(pop_result.status, Equals(StackPopResult::StackPopStoppedAtError)); @@ -224,14 +228,51 @@ describe("Stack", [&]() { free_slice_array(&pop_result.slices); }); }); + + describe("popping pending nodes from the stack", [&]() { + it("removes the top node from the stack if it was pushed in pending mode", [&]() { + ts_stack_push(stack, 0, trees[3], true, stateD); + + StackPopResult pop = ts_stack_pop_pending(stack, 0); + AssertThat(pop.status, Equals(StackPopResult::StackPopSucceeded)); + AssertThat(pop.slices.size, Equals(1)); + + AssertThat(get_stack_entries(stack, 0), Equals(vector({ + {stateC, 0}, + {stateB, 1}, + {stateA, 2}, + {0, 3}, + }))); + + free_slice_array(&pop.slices); + }); + + it("does nothing if the top node was not pushed in pending mode", [&]() { + ts_stack_push(stack, 0, trees[3], false, stateD); + + StackPopResult pop = ts_stack_pop_pending(stack, 0); + AssertThat(pop.status, Equals(StackPopResult::StackPopSucceeded)); + AssertThat(pop.slices.size, Equals(0)); + + AssertThat(get_stack_entries(stack, 0), Equals(vector({ + {stateD, 0}, + {stateC, 1}, + {stateB, 2}, + {stateA, 3}, + {0, 4}, + }))); + + free_slice_array(&pop.slices); + }); + }); }); describe("splitting the stack", [&]() { it("creates a new independent head with the same entries", [&]() { // . <──0── A <──1── B <──2── C* - ts_stack_push(stack, 0, trees[0], stateA); - ts_stack_push(stack, 0, trees[1], stateB); - ts_stack_push(stack, 0, trees[2], stateC); + ts_stack_push(stack, 0, trees[0], false, stateA); + ts_stack_push(stack, 0, trees[1], false, stateB); + ts_stack_push(stack, 0, trees[2], false, stateC); // . <──0── A <──1── B <──2── C* // ↑ @@ -244,7 +285,7 @@ describe("Stack", [&]() { // . <──0── A <──1── B <──2── C <──3── D* // ↑ // └─* - ts_stack_push(stack, 0, trees[3], stateD); + ts_stack_push(stack, 0, trees[3], false, stateD); StackPopResult pop_result = ts_stack_pop_count(stack, 1, 1); AssertThat(ts_stack_head_count(stack), Equals(2)); @@ -261,8 +302,8 @@ describe("Stack", [&]() { // . <──0── A <──1── B <──2── C <──3── D* // ↑ // └───4─── E <──5── F* - ts_stack_push(stack, 1, trees[4], stateE); - ts_stack_push(stack, 1, trees[5], stateF); + ts_stack_push(stack, 1, trees[4], false, stateE); + ts_stack_push(stack, 1, trees[5], false, stateF); AssertThat(ts_stack_head_count(stack), Equals(2)); AssertThat(ts_stack_top_state(stack, 0), Equals(stateD)); @@ -277,13 +318,13 @@ describe("Stack", [&]() { // . <──0── A <──1── B <──2── C <──3── D* // ↑ // └───4─── E <──5── F* - ts_stack_push(stack, 0, trees[0], stateA); - ts_stack_push(stack, 0, trees[1], stateB); + ts_stack_push(stack, 0, trees[0], false, stateA); + ts_stack_push(stack, 0, trees[1], false, stateB); ts_stack_split(stack, 0); - ts_stack_push(stack, 0, trees[2], stateC); - ts_stack_push(stack, 0, trees[3], stateD); - ts_stack_push(stack, 1, trees[4], stateE); - ts_stack_push(stack, 1, trees[5], stateF); + ts_stack_push(stack, 0, trees[2], false, stateC); + ts_stack_push(stack, 0, trees[3], false, stateD); + ts_stack_push(stack, 1, trees[4], false, stateE); + ts_stack_push(stack, 1, trees[5], false, stateF); AssertThat(ts_stack_head_count(stack), Equals(2)); AssertThat(get_stack_entries(stack, 0), Equals(vector({ @@ -306,8 +347,8 @@ describe("Stack", [&]() { // . <──0── A <──1── B <──2── C <──3── D <──6── G* // ↑ | // └───4─── E <──5── F <──7───┘ - AssertThat(ts_stack_push(stack, 0, trees[6], stateG), Equals(StackPushContinued)); - AssertThat(ts_stack_push(stack, 1, trees[7], stateG), Equals(StackPushMerged)); + AssertThat(ts_stack_push(stack, 0, trees[6], false, stateG), Equals(StackPushContinued)); + AssertThat(ts_stack_push(stack, 1, trees[7], false, stateG), Equals(StackPushMerged)); AssertThat(ts_stack_head_count(stack), Equals(1)); AssertThat(get_stack_entries(stack, 0), Equals(vector({ @@ -327,14 +368,14 @@ describe("Stack", [&]() { // . <──0── A <──1── B <──2── C <──3── D <──6── G <──7──H* // ↑ // └───4─── E <──5── F <──8── G* - AssertThat(ts_stack_push(stack, 0, trees[6], stateG), Equals(StackPushContinued)); - AssertThat(ts_stack_push(stack, 0, trees[7], stateH), Equals(StackPushContinued)); - AssertThat(ts_stack_push(stack, 1, trees[6], stateG), Equals(StackPushContinued)); + AssertThat(ts_stack_push(stack, 0, trees[6], false, stateG), Equals(StackPushContinued)); + AssertThat(ts_stack_push(stack, 0, trees[7], false, stateH), Equals(StackPushContinued)); + AssertThat(ts_stack_push(stack, 1, trees[6], false, stateG), Equals(StackPushContinued)); // . <──0── A <──1── B <──2── C <──3── D <──6── G <──7──H* // ↑ | // └───4─── E <──5── F <──8───┘ - AssertThat(ts_stack_push(stack, 1, trees[7], stateH), Equals(StackPushMerged)); + AssertThat(ts_stack_push(stack, 1, trees[7], false, stateH), Equals(StackPushMerged)); AssertThat(ts_stack_head_count(stack), Equals(1)); AssertThat(get_stack_entries(stack, 0), Equals(vector({ @@ -362,9 +403,9 @@ describe("Stack", [&]() { // └───2─── B ───3───┘ ts_stack_clear(stack); ts_stack_split(stack, 0); - AssertThat(ts_stack_push(stack, 0, parent, stateC), Equals(StackPushContinued)); - AssertThat(ts_stack_push(stack, 1, trees[2], stateB), Equals(StackPushContinued)); - AssertThat(ts_stack_push(stack, 1, trees[3], stateC), Equals(StackPushMerged)); + AssertThat(ts_stack_push(stack, 0, parent, false, stateC), Equals(StackPushContinued)); + AssertThat(ts_stack_push(stack, 1, trees[2], false, stateB), Equals(StackPushContinued)); + AssertThat(ts_stack_push(stack, 1, trees[3], false, stateC), Equals(StackPushMerged)); AssertThat(ts_stack_head_count(stack), Equals(1)); AssertThat(ts_stack_top_state(stack, 0), Equals(stateC)); @@ -385,15 +426,15 @@ describe("Stack", [&]() { // . <──0── A <──1── B <──2── C <──3── D <──4── E* // ↑ | // └───5─── F <──6── G <──7───┘ - ts_stack_push(stack, 0, trees[0], stateA); - ts_stack_push(stack, 0, trees[1], stateB); + ts_stack_push(stack, 0, trees[0], false, stateA); + ts_stack_push(stack, 0, trees[1], false, stateB); ts_stack_split(stack, 0); - ts_stack_push(stack, 0, trees[2], stateC); - ts_stack_push(stack, 0, trees[3], stateD); - ts_stack_push(stack, 0, trees[4], stateE); - ts_stack_push(stack, 1, trees[5], stateF); - ts_stack_push(stack, 1, trees[6], stateG); - ts_stack_push(stack, 1, trees[7], stateE); + ts_stack_push(stack, 0, trees[2], false, stateC); + ts_stack_push(stack, 0, trees[3], false, stateD); + ts_stack_push(stack, 0, trees[4], false, stateE); + ts_stack_push(stack, 1, trees[5], false, stateF); + ts_stack_push(stack, 1, trees[6], false, stateG); + ts_stack_push(stack, 1, trees[7], false, stateE); AssertThat(ts_stack_head_count(stack), Equals(1)); AssertThat(get_stack_entries(stack, 0), Equals(vector({ @@ -449,7 +490,7 @@ describe("Stack", [&]() { // . <──0── A <──1── B <──2── C <──3── D <──4── E <──8──H* // ↑ | // └───5─── F <──6── G <──7───┘ - AssertThat(ts_stack_push(stack, 0, trees[8], stateH), Equals(StackPushContinued)); + AssertThat(ts_stack_push(stack, 0, trees[8], false, stateH), Equals(StackPushContinued)); AssertThat(ts_stack_head_count(stack), Equals(1)); AssertThat(ts_stack_top_state(stack, 0), Equals(stateH)); @@ -518,19 +559,19 @@ describe("Stack", [&]() { // | | // └───7─── G <──8── H <──9───┘ ts_stack_clear(stack); - ts_stack_push(stack, 0, trees[0], stateA); + ts_stack_push(stack, 0, trees[0], false, stateA); ts_stack_split(stack, 0); ts_stack_split(stack, 1); - ts_stack_push(stack, 0, trees[1], stateB); - ts_stack_push(stack, 0, trees[2], stateC); - ts_stack_push(stack, 0, trees[3], stateD); - ts_stack_push(stack, 1, trees[4], stateE); - ts_stack_push(stack, 1, trees[5], stateF); - ts_stack_push(stack, 1, trees[6], stateD); - ts_stack_push(stack, 1, trees[7], stateG); - ts_stack_push(stack, 1, trees[8], stateH); - ts_stack_push(stack, 1, trees[9], stateD); - ts_stack_push(stack, 0, trees[10], stateI); + ts_stack_push(stack, 0, trees[1], false, stateB); + ts_stack_push(stack, 0, trees[2], false, stateC); + ts_stack_push(stack, 0, trees[3], false, stateD); + ts_stack_push(stack, 1, trees[4], false, stateE); + ts_stack_push(stack, 1, trees[5], false, stateF); + ts_stack_push(stack, 1, trees[6], false, stateD); + ts_stack_push(stack, 1, trees[7], false, stateG); + ts_stack_push(stack, 1, trees[8], false, stateH); + ts_stack_push(stack, 1, trees[9], false, stateD); + ts_stack_push(stack, 0, trees[10], false, stateI); AssertThat(ts_stack_head_count(stack), Equals(1)); AssertThat(get_stack_entries(stack, 0), Equals(vector({ diff --git a/src/runtime/parser.c b/src/runtime/parser.c index ed329f33..a881be44 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -46,7 +46,6 @@ struct LookaheadState { TSTree *reusable_subtree; size_t reusable_subtree_pos; - bool is_verifying; }; struct ErrorRepair { @@ -87,6 +86,12 @@ typedef enum { RepairNoneFound, } RepairResult; +typedef enum { + BreakdownFailed, + BreakdownPerformed, + BreakdownAborted, +} BreakdownResult; + /* * Private */ @@ -95,8 +100,7 @@ static StackSlice ts_parser__pop_one(TSParser *self, int head_index, int count) StackPopResult pop = ts_stack_pop_count(self->stack, head_index, count); if (pop.status != StackPopSucceeded) return (StackSlice){ - .head_index = -1, - .trees = array_new(), + .head_index = -1, .trees = array_new(), }; assert(pop.slices.size > 0); assert(pop.slices.contents[0].head_index == head_index); @@ -107,21 +111,19 @@ static StackSlice ts_parser__pop_one(TSParser *self, int head_index, int count) return pop.slices.contents[0]; } -static ParseActionResult ts_parser__breakdown_top_of_stack(TSParser *self, - int head) { +static BreakdownResult ts_parser__breakdown_top_of_stack(TSParser *self, + int head) { TSTree *last_child = NULL; + bool did_break_down = false; + bool is_still_pending = false; do { - StackPopResult pop = ts_stack_pop_count(self->stack, head, 1); + StackPopResult pop = ts_stack_pop_pending(self->stack, head); if (!pop.slices.size) - return ParseActionFailed; - assert(pop.slices.size > 0); - - /* - * Since only one entry (not counting extra trees) is being popped from the - * stack, there should only be one possible array of removed trees. - */ + break; + did_break_down = true; + is_still_pending = false; for (size_t i = 0; i < pop.slices.size; i++) { StackSlice slice = pop.slices.contents[i]; TreeArray removed_trees = slice.trees; @@ -134,6 +136,8 @@ static ParseActionResult ts_parser__breakdown_top_of_stack(TSParser *self, TSStateId state = ts_stack_top_state(self->stack, head_index); for (size_t j = 0; j < parent->child_count; j++) { last_child = parent->children[j]; + is_still_pending = last_child->child_count > 0; + if (last_child->symbol == ts_builtin_sym_error) { state = ts_parse_state_error; } else if (!last_child->extra) { @@ -145,15 +149,15 @@ static ParseActionResult ts_parser__breakdown_top_of_stack(TSParser *self, LOG("breakdown_push sym:%s, size:%lu", SYM_NAME(last_child->symbol), ts_tree_total_size(last_child).chars); - - last_push = ts_stack_push(self->stack, head_index, last_child, state); + last_push = ts_stack_push(self->stack, head_index, last_child, + is_still_pending, state); if (last_push == StackPushFailed) goto error; } for (size_t j = 1, count = slice.trees.size; j < count; j++) { TSTree *tree = slice.trees.contents[j]; - last_push = ts_stack_push(self->stack, head_index, tree, state); + last_push = ts_stack_push(self->stack, head_index, tree, false, state); if (last_push == StackPushFailed) goto error; } @@ -167,12 +171,12 @@ static ParseActionResult ts_parser__breakdown_top_of_stack(TSParser *self, ts_tree_release(removed_trees.contents[j]); array_delete(&removed_trees); } - } while (last_child && last_child->child_count > 0); + } while (last_child && is_still_pending); - return ParseActionUpdated; + return did_break_down ? BreakdownPerformed : BreakdownAborted; error: - return ParseActionFailed; + return BreakdownFailed; } static void ts_parser__pop_reusable_subtree(LookaheadState *state); @@ -263,10 +267,8 @@ static TSTree *ts_parser__get_next_lookahead(TSParser *self, int head) { } if (state->reusable_subtree->has_changes) { - if (state->is_verifying && state->reusable_subtree->child_count == 0) { + if (state->reusable_subtree->child_count == 0) ts_parser__breakdown_top_of_stack(self, head); - state->is_verifying = false; - } LOG("breakdown_changed sym:%s", SYM_NAME(state->reusable_subtree->symbol)); ts_parser__breakdown_reusable_subtree(state); @@ -339,7 +341,9 @@ static int ts_parser__select_tree(void *data, TSTree *left, TSTree *right) { static ParseActionResult ts_parser__shift(TSParser *self, int head, TSStateId parse_state, TSTree *lookahead) { - switch (ts_stack_push(self->stack, head, lookahead, parse_state)) { + bool is_verifying = lookahead->child_count > 0; + switch ( + ts_stack_push(self->stack, head, lookahead, is_verifying, parse_state)) { case StackPushFailed: return ParseActionFailed; case StackPushMerged: @@ -446,7 +450,8 @@ static ReduceResult ts_parser__reduce(TSParser *self, int head, TSSymbol symbol, } } - StackPushResult push = ts_stack_push(self->stack, new_head, parent, state); + StackPushResult push = + ts_stack_push(self->stack, new_head, parent, false, state); ts_tree_release(parent); switch (push) { case StackPushFailed: @@ -464,7 +469,8 @@ static ReduceResult ts_parser__reduce(TSParser *self, int head, TSSymbol symbol, for (size_t j = 0; j < trailing_extra_count; j++) { size_t index = slice.trees.size - trailing_extra_count + j; TSTree *tree = slice.trees.contents[index]; - StackPushResult push = ts_stack_push(self->stack, new_head, tree, state); + StackPushResult push = + ts_stack_push(self->stack, new_head, tree, false, state); ts_tree_release(tree); switch (push) { case StackPushFailed: @@ -492,7 +498,8 @@ error: static StackIterateAction ts_parser__error_repair_callback(void *payload, TSStateId state, size_t tree_count, - bool is_done) { + bool is_done, + bool is_pending) { ErrorRepairSession *session = (ErrorRepairSession *)payload; const TSParser *self = session->parser; size_t count_above_error = session->count_above_error; @@ -596,7 +603,7 @@ static RepairResult ts_parser__repair_error(TSParser *self, int head_index, ts_language_symbol_metadata(self->language, symbol)); StackPushResult push_result = - ts_stack_push(self->stack, head_index, parent, repair.next_state); + ts_stack_push(self->stack, head_index, parent, false, repair.next_state); ts_tree_release(parent); switch (push_result) { case StackPushFailed: @@ -626,9 +633,7 @@ static ParseActionResult ts_parser__start(TSParser *self, TSInput input, ts_parser__select_tree); LookaheadState lookahead_state = { - .reusable_subtree = previous_tree, - .reusable_subtree_pos = 0, - .is_verifying = false, + .reusable_subtree = previous_tree, .reusable_subtree_pos = 0, }; array_clear(&self->lookahead_states); array_push(&self->lookahead_states, lookahead_state); @@ -688,7 +693,7 @@ static ParseActionResult ts_parser__handle_error(TSParser *self, int head, TSTree *error = ts_tree_make_node( ts_builtin_sym_error, invalid_trees.size, invalid_trees.contents, ts_language_symbol_metadata(self->language, ts_builtin_sym_error)); - if (!ts_stack_push(self->stack, head, error, 0)) + if (!ts_stack_push(self->stack, head, error, false, 0)) goto error; StackSlice slice = ts_parser__pop_one(self, head, -1); @@ -698,7 +703,7 @@ static ParseActionResult ts_parser__handle_error(TSParser *self, int head, ts_builtin_sym_start, slice.trees.size, slice.trees.contents, ts_language_symbol_metadata(self->language, ts_builtin_sym_start)); - if (!ts_stack_push(self->stack, head, parent, 0)) + if (!ts_stack_push(self->stack, head, parent, false, 0)) goto error; ts_tree_release(parent); @@ -716,16 +721,17 @@ static ParseActionResult ts_parser__handle_error(TSParser *self, int head, assert(action.type == TSParseActionTypeShift); TSStateId next_state = action.data.to_state; - if (ts_language_has_action(self->language, next_state, following_token->symbol)) { + if (ts_language_has_action(self->language, next_state, + following_token->symbol)) { LOG_ACTION("resume_without_context state:%d", next_state); ts_lexer_reset(&self->lexer, position); TSTree *error = ts_tree_make_node( ts_builtin_sym_error, invalid_trees.size, invalid_trees.contents, ts_language_symbol_metadata(self->language, ts_builtin_sym_error)); error->extra = true; - if (!ts_parser__shift(self, head, ts_parse_state_error, error)) + if (!ts_stack_push(self->stack, head, error, false, ts_parse_state_error)) goto error; - if (!ts_parser__shift(self, head, next_state, next_token)) + if (!ts_stack_push(self->stack, head, next_token, false, next_state)) goto error; ts_tree_release(error); @@ -770,8 +776,8 @@ static ParseActionResult ts_parser__consume_lookahead(TSParser *self, int head, TSParseAction action = actions[i]; bool should_skip = repaired_error && - action.type == TSParseActionTypeReduce && - action.data.child_count > child_count_above_error; + action.type == TSParseActionTypeReduce && + action.data.child_count > child_count_above_error; int current_head; if (i == action_count - 1) { @@ -787,15 +793,15 @@ static ParseActionResult ts_parser__consume_lookahead(TSParser *self, int head, LOG_STACK(); - LookaheadState *lookahead_state = - array_get(&self->lookahead_states, current_head); - switch (action.type) { - case TSParseActionTypeError: - if (lookahead_state->is_verifying) { - ts_parser__breakdown_top_of_stack(self, current_head); - lookahead_state->is_verifying = false; - return ParseActionRemoved; + case TSParseActionTypeError: { + switch (ts_parser__breakdown_top_of_stack(self, current_head)) { + case BreakdownFailed: + return ParseActionFailed; + case BreakdownPerformed: + return ParseActionRemoved; + default: + break; } if (ts_stack_head_count(self->stack) == 1) { @@ -806,6 +812,7 @@ static ParseActionResult ts_parser__consume_lookahead(TSParser *self, int head, ts_parser__remove_head(self, current_head); return ParseActionRemoved; } + } case TSParseActionTypeShift: if (action.extra) { @@ -813,14 +820,11 @@ static ParseActionResult ts_parser__consume_lookahead(TSParser *self, int head, return ts_parser__shift_extra(self, current_head, state, lookahead); } else { LOG_ACTION("shift state:%u", action.data.to_state); - lookahead_state->is_verifying = (lookahead->child_count > 0); TSStateId state = action.data.to_state; return ts_parser__shift(self, current_head, state, lookahead); } case TSParseActionTypeReduce: - lookahead_state->is_verifying = false; - if (action.extra) { LOG_ACTION("reduce_extra sym:%s", SYM_NAME(action.data.symbol)); ts_parser__reduce(self, current_head, action.data.symbol, 1, true, diff --git a/src/runtime/stack.c b/src/runtime/stack.c index 37774741..ec935845 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -19,6 +19,7 @@ typedef struct StackNode StackNode; typedef struct { StackNode *node; TSTree *tree; + bool is_pending; } StackLink; struct StackNode { @@ -34,6 +35,7 @@ typedef struct { size_t extra_count; StackNode *node; bool is_done; + bool is_pending; } PopPath; typedef struct { @@ -86,8 +88,9 @@ static void stack_node_release(StackNode *self, StackNodeArray *pool) { } } -static StackNode *stack_node_new(StackNode *next, TSTree *tree, TSStateId state, - TSLength position, StackNodeArray *pool) { +static StackNode *stack_node_new(StackNode *next, TSTree *tree, bool is_pending, + TSStateId state, TSLength position, + StackNodeArray *pool) { StackNode *node; if (pool->size > 0) node = array_pop(pool); @@ -105,14 +108,15 @@ static StackNode *stack_node_new(StackNode *next, TSTree *tree, TSStateId state, if (next) { ts_tree_retain(tree); stack_node_retain(next); - node->successor_count = 1, node->successors[0] = (StackLink){ next, tree }; + node->successor_count = 1; + node->successors[0] = (StackLink){ next, tree, is_pending }; } return node; } static void stack_node_add_successor(StackNode *self, TSTree *new_tree, - StackNode *new_node) { + bool is_pending, StackNode *new_node) { for (int i = 0; i < self->successor_count; i++) { StackLink successor = self->successors[i]; if (successor.tree == new_tree) { @@ -122,7 +126,7 @@ static void stack_node_add_successor(StackNode *self, TSTree *new_tree, successor.node->state == new_node->state) { for (int j = 0; j < new_node->successor_count; j++) { stack_node_add_successor(successor.node, new_node->successors[j].tree, - new_node->successors[j].node); + is_pending, new_node->successors[j].node); } return; } @@ -133,7 +137,7 @@ static void stack_node_add_successor(StackNode *self, TSTree *new_tree, stack_node_retain(new_node); ts_tree_retain(new_tree); self->successors[self->successor_count++] = (StackLink){ - new_node, new_tree, + new_node, new_tree, is_pending, }; } } @@ -171,7 +175,7 @@ Stack *ts_stack_new() { goto error; self->base_node = - stack_node_new(NULL, NULL, 0, ts_length_zero(), &self->node_pool); + stack_node_new(NULL, NULL, false, 0, ts_length_zero(), &self->node_pool); stack_node_retain(self->base_node); if (!self->base_node) goto error; @@ -272,7 +276,7 @@ void ts_stack_remove_head(Stack *self, int head_index) { */ StackPushResult ts_stack_push(Stack *self, int head_index, TSTree *tree, - TSStateId state) { + bool is_pending, TSStateId state) { StackNode *current_head = *array_get(&self->heads, head_index); TSLength position = ts_length_add(current_head->position, ts_tree_total_size(tree)); @@ -281,14 +285,14 @@ StackPushResult ts_stack_push(Stack *self, int head_index, TSTree *tree, StackNode *prior_node = self->heads.contents[i]; if (prior_node->state == state && prior_node->position.chars == position.chars) { - stack_node_add_successor(prior_node, tree, current_head); + stack_node_add_successor(prior_node, tree, is_pending, current_head); ts_stack_remove_head(self, head_index); return StackPushMerged; } } - StackNode *new_head = - stack_node_new(current_head, tree, state, position, &self->node_pool); + StackNode *new_head = stack_node_new(current_head, tree, is_pending, state, + position, &self->node_pool); if (!new_head) return StackPushFailed; @@ -309,7 +313,11 @@ static inline ALWAYS_INLINE StackSliceArray stack__pop( StackNode *initial_head = *array_get(&self->heads, head_index); PopPath pop_path = { - .node = initial_head, .trees = array_new(), .extra_count = 0, .is_done = false, + .node = initial_head, + .trees = array_new(), + .extra_count = 0, + .is_done = false, + .is_pending = true, }; if (!array_grow(&pop_path.trees, STARTING_TREE_CAPACITY)) goto error; @@ -326,7 +334,8 @@ static inline ALWAYS_INLINE StackSliceArray stack__pop( StackNode *node = path->node; size_t successor_count = node->successor_count; - switch (callback(payload, node->state, depth - path->extra_count, node == self->base_node)) { + switch (callback(payload, node->state, depth - path->extra_count, + node == self->base_node, path->is_pending && depth > 0)) { case StackIteratePop: path->is_done = true; continue; @@ -365,6 +374,8 @@ static inline ALWAYS_INLINE StackSliceArray stack__pop( goto error; if (successor.tree->extra) next_path->extra_count++; + if (!successor.is_pending) + next_path->is_pending = false; ts_tree_retain(successor.tree); } } @@ -417,8 +428,9 @@ error: return self->slices; } -static inline ALWAYS_INLINE StackIterateAction stack__pop_count_callback( - void *payload, TSStateId state, size_t tree_count, bool is_done) { +static inline ALWAYS_INLINE StackIterateAction + stack__pop_count_callback(void *payload, TSStateId state, size_t tree_count, + bool is_done, bool is_pending) { StackPopSession *pop_session = (StackPopSession *)payload; if (pop_session->found_error) return StackIterateAbort; @@ -438,6 +450,20 @@ static inline ALWAYS_INLINE StackIterateAction stack__pop_count_callback( return StackIterateContinue; } +static inline ALWAYS_INLINE StackIterateAction + stack__pop_pending_callback(void *payload, TSStateId state, size_t tree_count, + bool is_done, bool is_pending) { + if (tree_count >= 1) { + if (is_pending) { + return StackIteratePop; + } else { + return StackIterateAbort; + } + } else { + return StackIterateContinue; + } +} + StackPopResult ts_stack_pop_count(Stack *self, int head_index, int count) { StackPopSession session = { .goal_tree_count = count, .found_error = false, @@ -471,6 +497,12 @@ StackPopResult ts_stack_pop_until(Stack *self, int head_index, return (StackPopResult){.status = StackPopSucceeded, .slices = slices }; } +StackPopResult ts_stack_pop_pending(Stack *self, int head_index) { + StackSliceArray slices = + stack__pop(self, head_index, stack__pop_pending_callback, NULL); + return (StackPopResult){.status = StackPopSucceeded, .slices = slices }; +} + void ts_stack_shrink(Stack *self, int head_index, int count) { StackNode *head = *array_get(&self->heads, head_index); StackNode *new_head = head; diff --git a/src/runtime/stack.h b/src/runtime/stack.h index e4961d25..3a1b6b2d 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -41,7 +41,8 @@ typedef enum { typedef StackIterateAction (*StackIterateCallback)(void *, TSStateId state, size_t tree_count, - bool is_done); + bool is_done, + bool is_pending); typedef int (*TreeSelectionFunction)(void *, TSTree *tree1, TSTree *tree2); @@ -76,7 +77,8 @@ TSLength ts_stack_top_position(const Stack *, int head_index); * Push a (tree, state) pair onto the given head of the stack. This could cause * the head to merge with an existing head. */ -StackPushResult ts_stack_push(Stack *, int head_index, TSTree *, TSStateId); +StackPushResult ts_stack_push(Stack *, int head_index, TSTree *, bool, + TSStateId); /* * Pop the given number of entries from the given head of the stack. This @@ -89,6 +91,8 @@ StackPopResult ts_stack_pop_count(Stack *, int head_index, int count); StackPopResult ts_stack_pop_until(Stack *, int head_index, StackIterateCallback, void *); +StackPopResult ts_stack_pop_pending(Stack *, int head_index); + /* * Remove the given number of entries from the given head of the stack. */