diff --git a/spec/runtime/stack_spec.cc b/spec/runtime/stack_spec.cc index 88041e95..47e55ec7 100644 --- a/spec/runtime/stack_spec.cc +++ b/spec/runtime/stack_spec.cc @@ -72,15 +72,15 @@ struct StackEntry { vector get_stack_entries(Stack *stack, StackVersion version) { vector result; - ts_stack_pop_until( + ts_stack_iterate( stack, version, - [](void *payload, TSStateId state, size_t tree_count, bool is_done, bool is_pending) { + [](void *payload, TSStateId state, size_t tree_count, bool is_done, bool is_pending) -> StackIterateAction { auto entries = static_cast *>(payload); StackEntry entry = {state, tree_count}; if (find(entries->begin(), entries->end(), entry) == entries->end()) entries->push_back(entry); - return StackIterateContinue; + return StackIterateNone; }, &result); return result; } diff --git a/src/compiler/build_tables/build_parse_table.cc b/src/compiler/build_tables/build_parse_table.cc index 0b74f8f8..9bd0b8fb 100644 --- a/src/compiler/build_tables/build_parse_table.cc +++ b/src/compiler/build_tables/build_parse_table.cc @@ -262,8 +262,8 @@ class ParseTableBuilder { } void remove_duplicate_parse_states() { - auto replacements = remove_duplicate_states( - &parse_table.states); + auto replacements = + remove_duplicate_states(&parse_table.states); for (auto &pair : parse_table.out_of_context_state_indices) { auto replacement = replacements.find(pair.second); diff --git a/src/compiler/build_tables/item_set_closure.cc b/src/compiler/build_tables/item_set_closure.cc index 4f890e98..b74431ba 100644 --- a/src/compiler/build_tables/item_set_closure.cc +++ b/src/compiler/build_tables/item_set_closure.cc @@ -37,7 +37,8 @@ ParseItemSet item_set_closure(const ParseItemSet &input_item_set, // If the next symbol in the production is not a non-terminal, skip to the // next item. Symbol next_symbol = item.next_symbol(); - if (next_symbol == NONE() || next_symbol.is_token || next_symbol.is_built_in()) + if (next_symbol == NONE() || next_symbol.is_token || + next_symbol.is_built_in()) continue; // If the next symbol is the last symbol in the item's production, then the diff --git a/src/compiler/generate_code/c_code.cc b/src/compiler/generate_code/c_code.cc index 4d7475b1..f407f7ff 100644 --- a/src/compiler/generate_code/c_code.cc +++ b/src/compiler/generate_code/c_code.cc @@ -228,8 +228,9 @@ class CCodeGenerator { if (symbol.is_built_in()) continue; auto iter = parse_table.out_of_context_state_indices.find(symbol); - string state = (iter != parse_table.out_of_context_state_indices.end()) ? - to_string(iter->second) : "ts_parse_state_error"; + string state = (iter != parse_table.out_of_context_state_indices.end()) + ? to_string(iter->second) + : "ts_parse_state_error"; line("[" + symbol_id(symbol) + "] = " + state + ","); } }); @@ -277,7 +278,8 @@ class CCodeGenerator { for (const ParseState &state : parse_table.states) { if (!state.in_progress_symbols.empty()) { line("[" + to_string(state_id) + "] = "); - add(to_string(add_in_progress_symbol_list_id(state.in_progress_symbols))); + add(to_string( + add_in_progress_symbol_list_id(state.in_progress_symbols))); add(","); } state_id++; diff --git a/src/runtime/array.h b/src/runtime/array.h index 239de3ae..47366c51 100644 --- a/src/runtime/array.h +++ b/src/runtime/array.h @@ -55,11 +55,13 @@ extern "C" { #define array_reverse(self) \ array__reverse((VoidArray *)(self), array__elem_size(self)) -#define array_copy(self) \ - { \ - memcpy(ts_calloc((self)->capacity, array__elem_size(self)), \ - (self)->contents, (self)->size *array__elem_size(self)), \ - (self)->size, (self)->capacity, \ +#define array_copy(self) \ + { \ + (self)->contents \ + ? memcpy(ts_calloc((self)->capacity, array__elem_size(self)), \ + (self)->contents, (self)->size *array__elem_size(self)) \ + : NULL, \ + (self)->size, (self)->capacity, \ } // Private diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 0de94c11..ef551a5c 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -52,18 +52,20 @@ typedef struct { } ReusableNode; struct ErrorRepair { - TSParseAction action; - TSStateId next_state; + TSSymbol symbol; size_t in_progress_state_count; - size_t essential_tree_count; + size_t count_below_error; }; typedef struct { - ErrorRepairArray *repairs; - size_t count_above_error; - int best_repair_index; TSParser *parser; TSSymbol lookahead_symbol; + ErrorRepairArray *repairs; + bool found_repair; + ErrorRepair best_repair; + TSStateId best_repair_state; + TSStateId best_repair_next_state; + size_t best_repair_skip_count; } ErrorRepairSession; typedef enum { @@ -384,28 +386,39 @@ static StackIterateAction ts_parser__error_repair_callback(void *payload, bool is_pending) { ErrorRepairSession *session = (ErrorRepairSession *)payload; const TSParser *self = session->parser; - size_t count_above_error = session->count_above_error; + const TSLanguage *language = self->language; TSSymbol lookahead_symbol = session->lookahead_symbol; - - if (session->best_repair_index >= 0) - return StackIterateAbort; + StackIterateAction result = StackIterateNone; for (size_t i = 0; i < session->repairs->size; i++) { ErrorRepair *repair = &session->repairs->contents[i]; - TSSymbol symbol = repair->action.data.symbol; - size_t child_count = repair->action.data.child_count; + TSSymbol symbol = repair->symbol; - if (tree_count + count_above_error >= child_count && - repair->in_progress_state_count > 0) { - TSParseAction new_action = - ts_language_last_action(self->language, state, symbol); - if (new_action.type == TSParseActionTypeShift) { - repair->next_state = new_action.data.to_state; - if (ts_language_has_action(self->language, repair->next_state, - lookahead_symbol)) { - session->best_repair_index = i; - repair->essential_tree_count = tree_count; - return StackIteratePop; + if (tree_count >= repair->count_below_error) { + size_t skip_count = tree_count - repair->count_below_error; + + if (session->found_repair && skip_count > session->best_repair_skip_count) { + array_erase(session->repairs, i--); + continue; + } + + if (repair->in_progress_state_count > 0) { + TSParseAction action = ts_language_last_action(language, state, symbol); + if (action.type == TSParseActionTypeShift) { + TSStateId next_state = action.data.to_state; + if (ts_language_has_action(language, next_state, lookahead_symbol) && + (!session->found_repair || + repair->in_progress_state_count > + session->best_repair.in_progress_state_count)) { + result |= StackIteratePop; + session->found_repair = true; + session->best_repair = *repair; + session->best_repair_state = state; + session->best_repair_skip_count = skip_count; + session->best_repair_next_state = next_state; + array_erase(session->repairs, i--); + continue; + } } } } @@ -416,75 +429,85 @@ static StackIterateAction ts_parser__error_repair_callback(void *payload, repair->in_progress_state_count = 0; } - return StackIterateContinue; + if (session->repairs->size == 0) + result |= StackIterateStop; + + return result; } static RepairResult ts_parser__repair_error(TSParser *self, StackSlice slice, TSTree *lookahead, const TSParseAction *actions, size_t action_count) { + size_t count_above_error = ts_tree_array_essential_count(&slice.trees); ErrorRepairSession session = { - .count_above_error = ts_tree_array_essential_count(&slice.trees), - .best_repair_index = -1, - .repairs = &self->error_repairs, - .lookahead_symbol = lookahead->symbol, .parser = self, + .lookahead_symbol = lookahead->symbol, + .repairs = &self->error_repairs, + .found_repair = false, }; array_clear(&self->error_repairs); for (size_t i = 0; i < action_count; i++) if (actions[i].type == TSParseActionTypeReduce && - actions[i].data.child_count > session.count_above_error) - CHECK(array_push(&self->error_repairs, ((ErrorRepair){ - .action = actions[i], - .in_progress_state_count = 0, - .essential_tree_count = 0, - }))); + actions[i].data.child_count > count_above_error) + CHECK(array_push( + &self->error_repairs, + ((ErrorRepair){ + .symbol = actions[i].data.symbol, + .count_below_error = actions[i].data.child_count - count_above_error, + .in_progress_state_count = 0, + }))); - StackPopResult pop = ts_stack_pop_until( + StackPopResult pop = ts_stack_iterate( self->stack, slice.version, ts_parser__error_repair_callback, &session); - if (!pop.slices.size) { + CHECK(pop.status); + if (!session.found_repair) { ts_tree_array_delete(&slice.trees); return RepairNoneFound; } - ts_stack_renumber_version(self->stack, pop.slices.contents[0].version, - slice.version); - ErrorRepair repair = self->error_repairs.contents[session.best_repair_index]; - size_t count_needed_below = - repair.action.data.child_count - session.count_above_error; - TreeArray children_below = pop.slices.contents[0].trees; - size_t count_skipped_below = repair.essential_tree_count - count_needed_below; - TSSymbol symbol = repair.action.data.symbol; + ErrorRepair repair = session.best_repair; + TSStateId next_state = session.best_repair_next_state; + size_t skip_count = session.best_repair_skip_count; + TSSymbol symbol = repair.symbol; + + StackSlice new_slice = array_pop(&pop.slices); + TreeArray children_below = new_slice.trees; + ts_stack_renumber_version(self->stack, new_slice.version, slice.version); + + while (pop.slices.size) { + StackSlice other_slice = array_pop(&pop.slices); + ts_tree_array_delete(&other_slice.trees); + ts_stack_remove_version(self->stack, other_slice.version); + } LOG_ACTION( - "repair_found sym:%s, child_count:%d, match_count:%lu, skipped:%lu", - SYM_NAME(symbol), repair.action.data.child_count, - repair.in_progress_state_count, count_skipped_below); + "repair_found sym:%s, child_count:%lu, match_count:%lu, skipped:%lu", + SYM_NAME(symbol), repair.count_below_error, repair.in_progress_state_count, + skip_count); - if (count_skipped_below > 0) { + if (skip_count > 0) { TreeArray skipped_children = array_new(); - CHECK(array_grow(&skipped_children, count_skipped_below)); - for (size_t i = count_needed_below; i < children_below.size; i++) + CHECK(array_grow(&skipped_children, skip_count)); + for (size_t i = repair.count_below_error; i < children_below.size; i++) array_push(&skipped_children, children_below.contents[i]); TSTree *error = ts_tree_make_error_node(&skipped_children); CHECK(error); - children_below.size = count_needed_below; + children_below.size = repair.count_below_error; array_push(&children_below, error); } for (size_t i = 0; i < slice.trees.size; i++) array_push(&children_below, slice.trees.contents[i]); - array_delete(&slice); + array_delete(&slice.trees); TSTree *parent = ts_tree_make_node(symbol, children_below.size, children_below.contents, ts_language_symbol_metadata(self->language, symbol)); CHECK(parent); - CHECK(ts_stack_push(self->stack, slice.version, parent, false, - repair.next_state)); - ts_tree_release(parent); + CHECK(ts_parser__push(self, slice.version, parent, next_state)); return RepairSucceeded; error: diff --git a/src/runtime/stack.c b/src/runtime/stack.c index 01cafba4..e20fe81b 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -11,7 +11,7 @@ #define STARTING_TREE_CAPACITY 10 #define MAX_NODE_POOL_SIZE 50 -#define ALWAYS_INLINE __attribute__((always_inline)) +#define INLINE static inline __attribute__((always_inline)) typedef struct StackNode StackNode; @@ -31,9 +31,8 @@ struct StackNode { typedef struct { TreeArray trees; - size_t essential_tree_count; + size_t tree_count; StackNode *node; - bool is_done; bool is_pending; } PopPath; @@ -58,10 +57,6 @@ static const char *COLORS[] = { "red", "blue", "orange", "green", "purple", }; -/* - * Section: Manipulating nodes (Private) - */ - static void stack_node_retain(StackNode *self) { if (!self) return; @@ -137,10 +132,6 @@ static void stack_node_add_link(StackNode *self, StackLink link) { } } -/* - * Section: Stack lifecycle - */ - static int ts_stack__default_tree_selection(void *p, TSTree *t1, TSTree *t2) { return 0; } @@ -194,10 +185,6 @@ error: return NULL; } -/* - * Section: Reading from the stack - */ - TSStateId ts_stack_top_state(const Stack *self, StackVersion version) { return (*array_get(&self->heads, version))->state; } @@ -210,39 +197,6 @@ size_t ts_stack_version_count(const Stack *self) { return self->heads.size; } -static void ts_stack__merge_slice(Stack *self, StackSlice *slice, - StackSlice *new_slice) { - bool should_update = false; - if (slice->trees.size < new_slice->trees.size) { - should_update = true; - } else if (slice->trees.size == new_slice->trees.size) { - for (size_t i = 0; i < slice->trees.size; i++) { - TSTree *tree = slice->trees.contents[i]; - TSTree *new_tree = new_slice->trees.contents[i]; - int comparison = self->tree_selection_function( - self->tree_selection_payload, tree, new_tree); - if (comparison < 0) { - break; - } else if (comparison > 0) { - should_update = true; - break; - } - } - } - - if (should_update) { - ts_tree_array_delete(&slice->trees); - slice->trees = new_slice->trees; - slice->trees.size = new_slice->trees.size; - } else { - ts_tree_array_delete(&new_slice->trees); - } -} - -/* - * Section: Mutating the stack (Private) - */ - static StackVersion ts_stack__add_version(Stack *self, StackNode *node) { if (!array_push(&self->heads, node)) return STACK_VERSION_NONE; @@ -256,9 +210,51 @@ void ts_stack_remove_version(Stack *self, StackVersion version) { array_erase(&self->heads, version); } -/* - * Section: Mutating the stack (Public) - */ +static void ts_stack__merge_slice(Stack *self, StackSlice *slice, + TreeArray *trees) { + bool should_update = false; + if (slice->trees.size < trees->size) { + should_update = true; + } else if (slice->trees.size == trees->size) { + for (size_t i = 0; i < slice->trees.size; i++) { + TSTree *tree = slice->trees.contents[i]; + TSTree *new_tree = trees->contents[i]; + int comparison = self->tree_selection_function( + self->tree_selection_payload, tree, new_tree); + if (comparison < 0) { + break; + } else if (comparison > 0) { + should_update = true; + break; + } + } + } + + if (should_update) { + ts_tree_array_delete(&slice->trees); + slice->trees = *trees; + } else { + ts_tree_array_delete(trees); + } +} + +static bool ts_stack__add_slice(Stack *self, size_t previous_version_count, + StackNode *node, TreeArray *trees) { + for (size_t i = 0; i < self->slices.size; i++) { + StackSlice *previous_slice = &self->slices.contents[i]; + size_t version_index = previous_version_count + i; + if (self->heads.contents[version_index] == node) { + ts_stack__merge_slice(self, previous_slice, trees); + return true; + } + } + + StackVersion version = ts_stack__add_version(self, node); + if (version == STACK_VERSION_NONE) + return false; + StackSlice slice = {.version = version, .trees = *trees }; + return array_push(&self->slices, slice); +} bool ts_stack_push(Stack *self, StackVersion version, TSTree *tree, bool is_pending, TSStateId state) { @@ -299,57 +295,53 @@ void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) { array_erase(&self->heads, v1); } -static inline ALWAYS_INLINE StackPopResult - stack__pop(Stack *self, StackVersion version, StackIterateCallback callback, - void *payload) { +INLINE StackPopResult stack__iter(Stack *self, StackVersion version, + StackIterateCallback callback, void *payload) { + size_t previous_version_count = self->heads.size; + array_clear(&self->slices); + PopPath pop_path = { .node = *array_get(&self->heads, version), .trees = array_new(), - .essential_tree_count = 0, - .is_done = false, + .tree_count = 0, .is_pending = true, }; array_clear(&self->pop_paths); if (!array_push(&self->pop_paths, pop_path)) goto error; - bool all_paths_done = false; - while (!all_paths_done) { - all_paths_done = true; - + while (self->pop_paths.size > 0) { for (size_t i = 0, size = self->pop_paths.size; i < size; i++) { PopPath *path = &self->pop_paths.contents[i]; - if (path->is_done) - continue; - StackNode *node = path->node; - size_t link_count = node->link_count; + bool is_done = node == self->base_node; - switch (callback(payload, node->state, path->essential_tree_count, - node == self->base_node, path->is_pending)) { - case StackIteratePop: - path->is_done = true; - continue; - case StackIterateAbort: - link_count = 0; - break; - default: - break; + StackIterateAction action = callback( + payload, node->state, path->tree_count, is_done, path->is_pending); + + bool should_pop = action & StackIteratePop; + bool should_stop = action & StackIterateStop || node->link_count == 0; + + if (should_pop) { + TreeArray trees = + should_stop ? path->trees : ts_tree_array_copy(&path->trees); + array_reverse(&trees); + if (!ts_stack__add_slice(self, previous_version_count, node, &trees)) + goto error; } - if (link_count == 0) { - ts_tree_array_delete(&path->trees); + if (should_stop) { + if (!should_pop) + ts_tree_array_delete(&path->trees); array_erase(&self->pop_paths, i); i--, size--; continue; } - all_paths_done = false; - - for (size_t j = 1; j <= link_count; j++) { + for (size_t j = 1; j <= node->link_count; j++) { PopPath *next_path; StackLink link; - if (j == link_count) { + if (j == node->link_count) { link = node->links[0]; next_path = &self->pop_paths.contents[i]; } else { @@ -364,8 +356,7 @@ static inline ALWAYS_INLINE StackPopResult if (!link.is_pending) next_path->is_pending = false; if (!link.tree->extra && link.tree->symbol != ts_builtin_sym_error) - next_path->essential_tree_count++; - + next_path->tree_count++; if (!array_push(&next_path->trees, link.tree)) goto error; ts_tree_retain(link.tree); @@ -373,36 +364,7 @@ static inline ALWAYS_INLINE StackPopResult } } - array_clear(&self->slices); - for (size_t i = 0; i < self->pop_paths.size; i++) { - PopPath *path = &self->pop_paths.contents[i]; - if (!path->is_done) - continue; - - StackSlice slice = {.trees = path->trees, .version = STACK_VERSION_NONE }; - array_reverse(&slice.trees); - - bool merged = false; - for (size_t j = 0; j < self->slices.size; j++) { - StackSlice *prior_slice = &self->slices.contents[j]; - StackNode *prior_node = self->heads.contents[prior_slice->version]; - if (prior_node == path->node) { - ts_stack__merge_slice(self, prior_slice, &slice); - merged = true; - break; - } - } - - if (!merged) { - slice.version = ts_stack__add_version(self, path->node); - if (slice.version == STACK_VERSION_NONE) - goto error; - if (!array_push(&self->slices, slice)) - goto error; - } - } - - return (StackPopResult){.status = StackPopSucceeded, .slices = self->slices }; + return (StackPopResult){ StackPopSucceeded, self->slices }; error: for (size_t i = 0; i < self->pop_paths.size; i++) @@ -411,33 +373,30 @@ error: return (StackPopResult){.status = StackPopFailed }; } -StackPopResult ts_stack_pop_until(Stack *self, StackVersion version, - StackIterateCallback callback, void *payload) { - return stack__pop(self, version, callback, payload); +StackPopResult ts_stack_iterate(Stack *self, StackVersion version, + StackIterateCallback callback, void *payload) { + return stack__iter(self, version, callback, payload); } -static inline ALWAYS_INLINE StackIterateAction - stack__pop_count_callback(void *payload, TSStateId state, size_t tree_count, - bool is_done, bool is_pending) { +INLINE StackIterateAction pop_count_callback(void *payload, TSStateId state, + size_t tree_count, bool is_done, + bool is_pending) { StackPopSession *pop_session = (StackPopSession *)payload; if (pop_session->found_error) - return StackIterateAbort; + return StackIterateStop; if (tree_count == pop_session->goal_tree_count) - return StackIteratePop; + return StackIteratePop | StackIterateStop; if (state == ts_parse_state_error) { pop_session->found_error = true; - return StackIteratePop; + return StackIteratePop | StackIterateStop; } - return StackIterateContinue; + return StackIterateNone; } StackPopResult ts_stack_pop_count(Stack *self, StackVersion version, size_t count) { StackPopSession session = {.goal_tree_count = count, .found_error = false }; - - StackPopResult pop = - stack__pop(self, version, stack__pop_count_callback, &session); - + StackPopResult pop = stack__iter(self, version, pop_count_callback, &session); if (pop.status && session.found_error) { pop.status = StackPopStoppedAtError; array_reverse(&pop.slices); @@ -447,27 +406,25 @@ StackPopResult ts_stack_pop_count(Stack *self, StackVersion version, ts_stack_remove_version(self, slice.version); } } - return pop; } -static inline ALWAYS_INLINE StackIterateAction - stack__pop_pending_callback(void *payload, TSStateId state, size_t tree_count, - bool is_done, bool is_pending) { +INLINE StackIterateAction pop_pending_callback(void *payload, TSStateId state, + size_t tree_count, bool is_done, + bool is_pending) { if (tree_count >= 1) { if (is_pending) { - return StackIteratePop; + return StackIteratePop | StackIterateStop; } else { - return StackIterateAbort; + return StackIterateStop; } } else { - return StackIterateContinue; + return StackIterateNone; } } StackPopResult ts_stack_pop_pending(Stack *self, StackVersion version) { - StackPopResult pop = - stack__pop(self, version, stack__pop_pending_callback, NULL); + StackPopResult pop = stack__iter(self, version, pop_pending_callback, NULL); if (pop.slices.size > 0) { ts_stack_renumber_version(self, pop.slices.contents[0].version, version); pop.slices.contents[0].version = version; @@ -475,14 +432,14 @@ StackPopResult ts_stack_pop_pending(Stack *self, StackVersion version) { return pop; } -static inline ALWAYS_INLINE StackIterateAction - stack__pop_all_callback(void *payload, TSStateId state, size_t tree_count, - bool is_done, bool is_pending) { - return is_done ? StackIteratePop : StackIterateContinue; +INLINE StackIterateAction pop_all_callback(void *payload, TSStateId state, + size_t tree_count, bool is_done, + bool is_pending) { + return is_done ? (StackIteratePop | StackIterateStop) : StackIterateNone; } TreeArray ts_stack_pop_all(Stack *self, StackVersion version) { - StackPopResult pop = stack__pop(self, version, stack__pop_all_callback, NULL); + StackPopResult pop = stack__iter(self, version, pop_all_callback, NULL); if (pop.status != StackPopSucceeded) return (TreeArray)array_new(); assert(pop.slices.size == 1); diff --git a/src/runtime/stack.h b/src/runtime/stack.h index 17c743c4..e01b6184 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -32,11 +32,13 @@ typedef struct { StackSliceArray slices; } StackPopResult; -typedef enum { - StackIterateContinue, - StackIterateAbort, - StackIteratePop, -} StackIterateAction; +enum { + StackIterateNone, + StackIterateStop = 1 << 0, + StackIteratePop = 1 << 1, +}; + +typedef unsigned int StackIterateAction; typedef StackIterateAction (*StackIterateCallback)(void *, TSStateId state, size_t tree_count, @@ -87,8 +89,8 @@ bool ts_stack_push(Stack *, StackVersion, TSTree *, bool, TSStateId); */ StackPopResult ts_stack_pop_count(Stack *, StackVersion, size_t count); -StackPopResult ts_stack_pop_until(Stack *, StackVersion, StackIterateCallback, - void *); +StackPopResult ts_stack_iterate(Stack *, StackVersion, StackIterateCallback, + void *); StackPopResult ts_stack_pop_pending(Stack *, StackVersion);