diff --git a/src/runtime/error_costs.c b/src/runtime/error_costs.c index 5e038b81..ac055f45 100644 --- a/src/runtime/error_costs.c +++ b/src/runtime/error_costs.c @@ -1,28 +1,46 @@ #include "runtime/error_costs.h" -#include -static inline unsigned error_status__min_cost(ErrorStatus status) { - return status.cost + ERROR_COST_PER_SKIPPED_TREE * status.count * status.count; -} - -static inline unsigned error_status__max_cost(ErrorStatus status) { - return status.cost + - ERROR_COST_PER_SKIPPED_TREE * status.count * status.count + - (6 * ERROR_COST_PER_SKIPPED_TREE * status.count + - 12 * ERROR_COST_PER_SKIPPED_TREE) / - (1 + status.push_count / 2); -} - -int error_status_compare(ErrorStatus a, ErrorStatus b) { - if ((a.count + 1 < b.count) || (a.count < b.count && a.cost <= b.cost)) - return -1; - if ((a.count > b.count + 1) || (b.count < a.count && b.cost <= a.cost)) - return 1; - - if (error_status__max_cost(a) < error_status__min_cost(b)) - return -1; - if (error_status__min_cost(a) > error_status__max_cost(b)) - return 1; - - return 0; +static const unsigned MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE; +static const unsigned MAX_PUSH_COUNT_WITH_COUNT_DIFFERENCE = 24; + +ErrorComparison error_status_compare(ErrorStatus a, ErrorStatus b, bool are_mergeable) { + if (a.count < b.count) { + if (are_mergeable || + a.cost <= b.cost || + a.count + 1 < b.count || + b.push_count > MAX_PUSH_COUNT_WITH_COUNT_DIFFERENCE) { + return ErrorComparisonTakeLeft; + } else { + return ErrorComparisonPreferLeft; + } + } + + if (b.count < a.count) { + if (are_mergeable || + b.cost <= a.cost || + b.count + 1 < a.count || + a.push_count > MAX_PUSH_COUNT_WITH_COUNT_DIFFERENCE) { + return ErrorComparisonTakeRight; + } else { + return ErrorComparisonPreferRight; + } + } + + if (a.cost < b.cost) { + if (are_mergeable || (b.cost - a.cost) * (1 + a.push_count) > MAX_COST_DIFFERENCE) { + return ErrorComparisonTakeLeft; + } else { + return ErrorComparisonPreferLeft; + } + } + + if (b.cost < a.cost) { + if (are_mergeable || (a.cost - b.cost) * (1 + b.push_count) > MAX_COST_DIFFERENCE) { + return ErrorComparisonTakeRight; + } else { + return ErrorComparisonPreferRight; + } + } + + return ErrorComparisonNone; } diff --git a/src/runtime/error_costs.h b/src/runtime/error_costs.h index aa5f3a92..968c2422 100644 --- a/src/runtime/error_costs.h +++ b/src/runtime/error_costs.h @@ -1,14 +1,16 @@ #ifndef RUNTIME_ERROR_COSTS_H_ #define RUNTIME_ERROR_COSTS_H_ +#include + #ifdef __cplusplus extern "C" { #endif #define ERROR_STATE 0 -#define ERROR_COST_PER_SKIPPED_TREE 10 -#define ERROR_COST_PER_SKIPPED_LINE 3 -#define ERROR_COST_PER_SKIPPED_CHAR 0 +#define ERROR_COST_PER_SKIPPED_TREE 100 +#define ERROR_COST_PER_SKIPPED_LINE 30 +#define ERROR_COST_PER_SKIPPED_CHAR 1 typedef struct { unsigned count; @@ -16,7 +18,15 @@ typedef struct { unsigned push_count; } ErrorStatus; -int error_status_compare(ErrorStatus a, ErrorStatus b); +typedef enum { + ErrorComparisonTakeLeft, + ErrorComparisonPreferLeft, + ErrorComparisonNone, + ErrorComparisonPreferRight, + ErrorComparisonTakeRight, +} ErrorComparison; + +ErrorComparison error_status_compare(ErrorStatus a, ErrorStatus b, bool can_merge); #ifdef __cplusplus } diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 492efc18..b77ec2b3 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -40,6 +40,8 @@ #define SYM_NAME(symbol) ts_language_symbol_name(self->language, symbol) +static const uint32_t MAX_STACK_VERSION_COUNT = 16; + typedef struct { Parser *parser; TSSymbol lookahead_symbol; @@ -166,31 +168,55 @@ static CondenseResult parser__condense_stack(Parser *self) { continue; } - ErrorStatus error_status = ts_stack_error_status(self->stack, i); - if (error_status.count == 0) has_version_without_errors = true; + StackVersion version_to_swap = STACK_VERSION_NONE; + ErrorStatus right_error_status = ts_stack_error_status(self->stack, i); + if (right_error_status.count == 0) has_version_without_errors = true; for (StackVersion j = 0; j < i; j++) { - if (ts_stack_merge(self->stack, j, i)) { - result |= CondenseResultMadeChange; - i--; - break; - } + bool can_merge = ts_stack_can_merge(self->stack, i, j); + ErrorStatus left_error_status = ts_stack_error_status(self->stack, j); - switch (error_status_compare(error_status, - ts_stack_error_status(self->stack, j))) { - case -1: + switch (error_status_compare(left_error_status, right_error_status, can_merge)) { + case ErrorComparisonTakeLeft: + ts_stack_remove_version(self->stack, i); + result |= CondenseResultMadeChange; + i--; + j = i; + break; + + case ErrorComparisonTakeRight: ts_stack_remove_version(self->stack, j); result |= CondenseResultMadeChange; i--; j--; break; - case 1: - ts_stack_remove_version(self->stack, i); + + case ErrorComparisonPreferRight: + if (version_to_swap != STACK_VERSION_NONE) version_to_swap = j; result |= CondenseResultMadeChange; - i--; break; + + case ErrorComparisonPreferLeft: + break; + + case ErrorComparisonNone: + if (can_merge) { + ts_stack_force_merge(self->stack, j, i); + result |= CondenseResultMadeChange; + i--; + break; + } } } + + if (version_to_swap != STACK_VERSION_NONE) { + ts_stack_swap_versions(self->stack, i, version_to_swap); + } + } + + while (ts_stack_version_count(self->stack) > MAX_STACK_VERSION_COUNT) { + ts_stack_remove_version(self->stack, MAX_STACK_VERSION_COUNT); + result |= CondenseResultMadeChange; } if (!has_version_without_errors && ts_stack_version_count(self->stack) > 0) { @@ -454,13 +480,16 @@ static bool parser__better_version_exists(Parser *self, StackVersion version, continue; switch (error_status_compare(my_error_status, - ts_stack_error_status(self->stack, i))) { - case -1: + ts_stack_error_status(self->stack, i), + ts_stack_can_merge(self->stack, i, version))) { + case ErrorComparisonTakeLeft: LOG("halt_other version:%u", i); ts_stack_halt(self->stack, i); break; - case 1: + case ErrorComparisonTakeRight: return true; + default: + break; } } @@ -605,10 +634,15 @@ static StackPopResult parser__reduce(Parser *self, StackVersion version, return pop; } -static inline const TSParseAction *parser__reductions_after_sequence( - Parser *self, TSStateId start_state, const TreeArray *trees_below, - uint32_t tree_count_below, const TreeArray *trees_above, - TSSymbol lookahead_symbol, uint32_t *count) { +static const TSParseAction *parser__reductions_after_sequence( + Parser *self, + TSStateId start_state, + const TreeArray *trees_below, + uint32_t tree_count_below, + const TreeArray *trees_above, + TSSymbol lookahead_symbol, + uint32_t *count +) { TSStateId state = start_state; uint32_t child_count = 0; *count = 0; @@ -658,11 +692,10 @@ static inline const TSParseAction *parser__reductions_after_sequence( return actions; } -static StackIterateAction parser__repair_error_callback( - void *payload, TSStateId state, TreeArray *trees, uint32_t tree_count, - bool is_done, bool is_pending) { - - ErrorRepairSession *session = (ErrorRepairSession *)payload; +static StackIterateAction parser__repair_error_callback(void *payload, TSStateId state, + const TreeArray *trees, + uint32_t tree_count) { + ErrorRepairSession *session = payload; Parser *self = session->parser; TSSymbol lookahead_symbol = session->lookahead_symbol; ReduceActionSet *repairs = &self->reduce_actions; @@ -928,8 +961,7 @@ static bool parser__do_potential_reductions(Parser *self, StackVersion version) } static StackIterateAction parser__skip_preceding_trees_callback( - void *payload, TSStateId state, TreeArray *trees, uint32_t tree_count, - bool is_done, bool is_pending) { + void *payload, TSStateId state, const TreeArray *trees, uint32_t tree_count) { if (tree_count > 0 && state != ERROR_STATE) { uint32_t bytes_skipped = 0; for (uint32_t i = 0; i < trees->size; i++) { @@ -1013,7 +1045,7 @@ static void parser__handle_error(Parser *self, StackVersion version, ts_stack_push(self->stack, version, NULL, false, ERROR_STATE); while (ts_stack_version_count(self->stack) > previous_version_count) { ts_stack_push(self->stack, previous_version_count, NULL, false, ERROR_STATE); - assert(ts_stack_merge(self->stack, version, previous_version_count)); + ts_stack_force_merge(self->stack, version, previous_version_count); } } @@ -1049,21 +1081,13 @@ static void parser__recover(Parser *self, StackVersion version, TSStateId state, Tree *parent = ts_tree_make_error_node(&children); parser__push(self, version, parent, 1); parser__accept(self, version, lookahead); + return; } LOG("recover state:%u", state); - StackVersion new_version = ts_stack_copy_version(self->stack, version); - - parser__shift( - self, new_version, ERROR_STATE, lookahead, - ts_language_symbol_metadata(self->language, lookahead->symbol).extra); - ErrorStatus error_status = ts_stack_error_status(self->stack, new_version); - if (parser__better_version_exists(self, version, error_status)) { - ts_stack_remove_version(self->stack, new_version); - LOG("bail_on_recovery"); - } - + bool can_be_extra = ts_language_symbol_metadata(self->language, lookahead->symbol).extra; + parser__shift(self, new_version, ERROR_STATE, lookahead, can_be_extra); parser__shift(self, version, state, lookahead, false); } diff --git a/src/runtime/stack.c b/src/runtime/stack.c index 5926ed78..d244fab0 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -10,14 +10,15 @@ #define MAX_NODE_POOL_SIZE 50 #define MAX_ITERATOR_COUNT 64 -#define INLINE static inline __attribute__((always_inline)) +#define inline static inline __attribute__((always_inline)) typedef struct StackNode StackNode; typedef struct { StackNode *node; Tree *tree; - unsigned push_count; + uint32_t push_count; + uint32_t depth; bool is_pending; } StackLink; @@ -32,11 +33,12 @@ struct StackNode { }; typedef struct { + StackNode *node; TreeArray trees; uint32_t tree_count; - StackNode *node; + uint32_t push_count; + uint32_t depth; bool is_pending; - unsigned push_count; } Iterator; typedef struct { @@ -45,13 +47,19 @@ typedef struct { bool found_valid_path; } StackPopSession; +typedef struct { + void *payload; + StackIterateCallback callback; +} StackIterateSession; + typedef Array(StackNode *) StackNodeArray; typedef struct { StackNode *node; - bool is_halted; - unsigned push_count; Tree *last_external_token; + uint32_t push_count; + uint32_t depth; + bool is_halted; } StackHead; struct Stack { @@ -62,6 +70,8 @@ struct Stack { StackNode *base_node; }; +typedef StackIterateAction (*StackIterateInternalCallback)(void *, const Iterator *); + static void stack_node_retain(StackNode *self) { if (!self) return; @@ -90,51 +100,44 @@ static void stack_node_release(StackNode *self, StackNodeArray *pool) { } } -static StackNode *stack_node_new(StackNode *next, Tree *tree, bool is_pending, - TSStateId state, Length position, - StackNodeArray *pool) { - StackNode *node; - if (pool->size > 0) - node = array_pop(pool); - else if (!(node = ts_malloc(sizeof(StackNode)))) - return NULL; +static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_pending, + TSStateId state, StackNodeArray *pool) { + StackNode *node = pool->size > 0 ? + array_pop(pool) : + ts_malloc(sizeof(StackNode)); + *node = (StackNode){.ref_count = 1, .link_count = 0, .state = state}; - *node = (StackNode){ - .ref_count = 1, - .link_count = 0, - .links = {}, - .state = state, - .position = position, - }; - - if (next) { - stack_node_retain(next); + if (previous_node) { + stack_node_retain(previous_node); node->link_count = 1; node->links[0] = (StackLink){ - .node = next, .tree = tree, .is_pending = is_pending, .push_count = 0, + .node = previous_node, + .tree = tree, + .is_pending = is_pending, + .push_count = 0, + .depth = 0, }; - node->error_count = next->error_count; - node->error_cost = next->error_cost; + node->position = previous_node->position; + node->error_count = previous_node->error_count; + node->error_cost = previous_node->error_cost; if (tree) { ts_tree_retain(tree); node->error_cost += tree->error_cost; - - if (state == ERROR_STATE) { - if (!tree->extra) { - node->error_cost += ERROR_COST_PER_SKIPPED_TREE + - ERROR_COST_PER_SKIPPED_CHAR * - (tree->padding.chars + tree->size.chars) + - ERROR_COST_PER_SKIPPED_LINE * - (tree->padding.extent.row + tree->size.extent.row); - } + node->position = length_add(node->position, ts_tree_total_size(tree)); + if (state == ERROR_STATE && !tree->extra) { + node->error_cost += + ERROR_COST_PER_SKIPPED_TREE * (tree->visible ? 1 : tree->visible_child_count) + + ERROR_COST_PER_SKIPPED_CHAR * (tree->padding.chars + tree->size.chars) + + ERROR_COST_PER_SKIPPED_LINE * (tree->padding.extent.row + tree->size.extent.row); } } else { node->error_count++; } } else { + node->position = length_zero(); node->error_count = 0; node->error_cost = 0; } @@ -142,17 +145,27 @@ static StackNode *stack_node_new(StackNode *next, Tree *tree, bool is_pending, return node; } +static bool stack__tree_is_equivalent(const Tree *left, const Tree *right) { + return left == right || ( + left && + right && + left->child_count == 0 && right->child_count == 0 && + left->symbol == right->symbol && + left->padding.bytes == right->padding.bytes && + left->size.bytes == right->size.bytes && + left->extra == right->extra && + ts_tree_external_token_state_eq(left, right)); +} + static void stack_node_add_link(StackNode *self, StackLink link) { for (int i = 0; i < self->link_count; i++) { StackLink existing_link = self->links[i]; - if (existing_link.tree == link.tree || - (existing_link.tree && link.tree && - ts_tree_tokens_eq(existing_link.tree, link.tree))) { - if (existing_link.node == link.node) - return; + if (stack__tree_is_equivalent(existing_link.tree, link.tree)) { + if (existing_link.node == link.node) return; if (existing_link.node->state == link.node->state) { - for (int j = 0; j < link.node->link_count; j++) + for (int j = 0; j < link.node->link_count; j++) { stack_node_add_link(existing_link.node, link.node->links[j]); + } return; } } @@ -160,31 +173,29 @@ static void stack_node_add_link(StackNode *self, StackLink link) { if (self->link_count < MAX_LINK_COUNT) { stack_node_retain(link.node); - if (link.tree) - ts_tree_retain(link.tree); - - self->links[self->link_count++] = (StackLink){ - .node = link.node, - .tree = link.tree, - .is_pending = link.is_pending, - .push_count = link.push_count, - }; + if (link.tree) ts_tree_retain(link.tree); + self->links[self->link_count++] = link; } } static void stack_head_delete(StackHead *self, StackNodeArray *pool) { - if (self->last_external_token) ts_tree_release(self->last_external_token); - stack_node_release(self->node, pool); + if (self->node) { + if (self->last_external_token) { + ts_tree_release(self->last_external_token); + } + stack_node_release(self->node, pool); + } } static StackVersion ts_stack__add_version(Stack *self, StackNode *node, - unsigned push_count, + uint32_t push_count, uint32_t depth, Tree *last_external_token) { StackHead head = { .node = node, - .is_halted = false, + .depth = depth, .push_count = push_count, .last_external_token = last_external_token, + .is_halted = false, }; array_push(&self->heads, head); stack_node_retain(node); @@ -193,7 +204,7 @@ static StackVersion ts_stack__add_version(Stack *self, StackNode *node, } static void ts_stack__add_slice(Stack *self, StackNode *node, TreeArray *trees, - unsigned push_count, + uint32_t push_count, uint32_t depth, Tree *last_external_token) { for (uint32_t i = self->slices.size - 1; i + 1 > 0; i--) { StackVersion version = self->slices.contents[i].version; @@ -204,18 +215,18 @@ static void ts_stack__add_slice(Stack *self, StackNode *node, TreeArray *trees, } } - StackVersion version = ts_stack__add_version(self, node, push_count, last_external_token); + StackVersion version = ts_stack__add_version(self, node, push_count, depth, last_external_token); StackSlice slice = { *trees, version }; array_push(&self->slices, slice); } -INLINE StackPopResult stack__iter(Stack *self, StackVersion version, - StackIterateCallback callback, void *payload) { +inline StackPopResult stack__iter(Stack *self, StackVersion version, + StackIterateInternalCallback callback, void *payload) { array_clear(&self->slices); array_clear(&self->iterators); StackHead *head = array_get(&self->heads, version); - unsigned push_count = head->push_count; + uint32_t starting_push_count = head->push_count; Tree *last_external_token = head->last_external_token; Iterator iterator = { .node = head->node, @@ -223,6 +234,7 @@ INLINE StackPopResult stack__iter(Stack *self, StackVersion version, .tree_count = 0, .is_pending = true, .push_count = 0, + .depth = head->depth, }; array_push(&self->iterators, iterator); @@ -230,12 +242,8 @@ INLINE StackPopResult stack__iter(Stack *self, StackVersion version, for (uint32_t i = 0, size = self->iterators.size; i < size; i++) { Iterator *iterator = &self->iterators.contents[i]; StackNode *node = iterator->node; - bool is_done = node == self->base_node; - - StackIterateAction action = - callback(payload, node->state, &iterator->trees, iterator->tree_count, - is_done, iterator->is_pending); + StackIterateAction action = callback(payload, iterator); bool should_pop = action & StackIteratePop; bool should_stop = action & StackIterateStop || node->link_count == 0; @@ -244,8 +252,14 @@ INLINE StackPopResult stack__iter(Stack *self, StackVersion version, if (!should_stop) ts_tree_array_copy(trees, &trees); array_reverse(&trees); - ts_stack__add_slice(self, node, &trees, push_count + iterator->push_count, - last_external_token); + ts_stack__add_slice( + self, + node, + &trees, + starting_push_count + iterator->push_count, + iterator->depth, + last_external_token + ); } if (should_stop) { @@ -273,9 +287,13 @@ INLINE StackPopResult stack__iter(Stack *self, StackVersion version, next_iterator->node = link.node; next_iterator->push_count += link.push_count; + if (link.depth > 0) { + next_iterator->depth = link.depth; + } if (link.tree) { if (!link.tree->extra) { next_iterator->tree_count++; + next_iterator->depth--; if (!link.is_pending) next_iterator->is_pending = false; } @@ -303,15 +321,8 @@ Stack *ts_stack_new() { array_grow(&self->iterators, 4); array_grow(&self->node_pool, MAX_NODE_POOL_SIZE); - self->base_node = - stack_node_new(NULL, NULL, false, 1, length_zero(), &self->node_pool); - stack_node_retain(self->base_node); - array_push(&self->heads, ((StackHead){ - self->base_node, - false, - 0, - NULL - })); + self->base_node = stack_node_new(NULL, NULL, false, 1, &self->node_pool); + ts_stack_clear(self); return self; } @@ -381,43 +392,43 @@ unsigned ts_stack_error_count(const Stack *self, StackVersion version) { return node->error_count; } -bool ts_stack_push(Stack *self, StackVersion version, Tree *tree, +void ts_stack_push(Stack *self, StackVersion version, Tree *tree, bool is_pending, TSStateId state) { StackHead *head = array_get(&self->heads, version); - StackNode *node = head->node; - Length position = node->position; - if (tree) - position = length_add(position, ts_tree_total_size(tree)); - StackNode *new_node = - stack_node_new(node, tree, is_pending, state, position, &self->node_pool); - if (!new_node) - return false; - stack_node_release(node, &self->node_pool); - head->node = new_node; + StackNode *new_node = stack_node_new(head->node, tree, is_pending, state, &self->node_pool); if (state == ERROR_STATE) { new_node->links[0].push_count = head->push_count; + new_node->links[0].depth = head->depth; head->push_count = 0; - } else + head->depth = 0; + } else { head->push_count++; - return true; + if (!tree->extra) head->depth++; + } + stack_node_release(head->node, &self->node_pool); + head->node = new_node; +} + +inline StackIterateAction iterate_callback(void *payload, const Iterator *iterator) { + StackIterateSession *session = payload; + return session->callback(session->payload, iterator->node->state, &iterator->trees, iterator->tree_count); } StackPopResult ts_stack_iterate(Stack *self, StackVersion version, StackIterateCallback callback, void *payload) { - return stack__iter(self, version, callback, payload); + StackIterateSession session = {payload, callback}; + return stack__iter(self, version, iterate_callback, &session); } -INLINE StackIterateAction pop_count_callback(void *payload, TSStateId state, - TreeArray *trees, uint32_t tree_count, - bool is_done, bool is_pending) { +inline StackIterateAction pop_count_callback(void *payload, const Iterator *iterator) { StackPopSession *pop_session = (StackPopSession *)payload; - if (tree_count == pop_session->goal_tree_count) { + if (iterator->tree_count == pop_session->goal_tree_count) { pop_session->found_valid_path = true; return StackIteratePop | StackIterateStop; } - if (state == ERROR_STATE) { + if (iterator->node->state == ERROR_STATE) { if (pop_session->found_valid_path || pop_session->found_error) { return StackIterateStop; } else { @@ -431,9 +442,13 @@ INLINE StackIterateAction pop_count_callback(void *payload, TSStateId state, StackPopResult ts_stack_pop_count(Stack *self, StackVersion version, uint32_t count) { StackPopSession session = { - .goal_tree_count = count, .found_error = false, .found_valid_path = false, + .goal_tree_count = count, + .found_error = false, + .found_valid_path = false, }; + StackPopResult pop = stack__iter(self, version, pop_count_callback, &session); + if (session.found_error) { if (session.found_valid_path) { StackSlice error_slice = pop.slices.contents[0]; @@ -441,22 +456,21 @@ StackPopResult ts_stack_pop_count(Stack *self, StackVersion version, array_erase(&pop.slices, 0); if (array_front(&pop.slices)->version != error_slice.version) { ts_stack_remove_version(self, error_slice.version); - for (StackVersion i = 0; i < pop.slices.size; i++) + for (StackVersion i = 0; i < pop.slices.size; i++) { pop.slices.contents[i].version--; + } } } else { pop.stopped_at_error = true; } } + return pop; } -INLINE StackIterateAction pop_pending_callback(void *payload, TSStateId state, - TreeArray *trees, - uint32_t tree_count, bool is_done, - bool is_pending) { - if (tree_count >= 1) { - if (is_pending) { +inline StackIterateAction pop_pending_callback(void *payload, const Iterator *iterator) { + if (iterator->tree_count >= 1) { + if (iterator->is_pending) { return StackIteratePop | StackIterateStop; } else { return StackIterateStop; @@ -475,10 +489,8 @@ StackPopResult ts_stack_pop_pending(Stack *self, StackVersion version) { return pop; } -INLINE StackIterateAction pop_all_callback(void *payload, TSStateId state, - TreeArray *trees, uint32_t tree_count, - bool is_done, bool is_pending) { - return is_done ? (StackIteratePop | StackIterateStop) : StackIterateNone; +inline StackIterateAction pop_all_callback(void *payload, const Iterator *iterator) { + return iterator->node->link_count == 0 ? StackIteratePop : StackIterateNone; } StackPopResult ts_stack_pop_all(Stack *self, StackVersion version) { @@ -498,6 +510,12 @@ void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) { array_erase(&self->heads, v1); } +void ts_stack_swap_versions(Stack *self, StackVersion v1, StackVersion v2) { + StackHead temporary_head = self->heads.contents[v1]; + self->heads.contents[v1] = self->heads.contents[v2]; + self->heads.contents[v2] = temporary_head; +} + StackVersion ts_stack_copy_version(Stack *self, StackVersion version) { assert(version < self->heads.size); array_push(&self->heads, self->heads.contents[version]); @@ -507,28 +525,38 @@ StackVersion ts_stack_copy_version(Stack *self, StackVersion version) { return self->heads.size - 1; } -bool ts_stack_merge(Stack *self, StackVersion version, StackVersion new_version) { - StackHead *head = &self->heads.contents[version]; - StackHead *new_head = &self->heads.contents[new_version]; - StackNode *node = head->node; - StackNode *new_node = new_head->node; - - if (new_node->state == node->state && - new_node->position.chars == node->position.chars && - new_node->error_count == node->error_count && - new_node->error_cost == node->error_cost && - ts_tree_external_token_state_eq(new_head->last_external_token, head->last_external_token)) { - for (uint32_t j = 0; j < new_node->link_count; j++) - stack_node_add_link(node, new_node->links[j]); - if (new_head->push_count > head->push_count) - head->push_count = new_head->push_count; - ts_stack_remove_version(self, new_version); +bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2) { + if (ts_stack_can_merge(self, version1, version2)) { + ts_stack_force_merge(self, version1, version2); return true; } else { return false; } } +bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version2) { + StackHead *head1 = &self->heads.contents[version1]; + StackHead *head2 = &self->heads.contents[version2]; + return + head1->node->state == head2->node->state && + head1->node->position.chars == head2->node->position.chars && + ts_tree_external_token_state_eq(head1->last_external_token, head2->last_external_token) && + ((head1->node->error_count == 0 && head2->node->error_count == 0) || + (head1->depth == head2->depth)); +} + +void ts_stack_force_merge(Stack *self, StackVersion version1, StackVersion version2) { + StackHead *head1 = &self->heads.contents[version1]; + StackHead *head2 = &self->heads.contents[version2]; + for (uint32_t i = 0; i < head2->node->link_count; i++) { + stack_node_add_link(head1->node, head2->node->links[i]); + } + if (head2->push_count > head1->push_count) { + head1->push_count = head2->push_count; + } + ts_stack_remove_version(self, version2); +} + void ts_stack_halt(Stack *self, StackVersion version) { array_get(&self->heads, version)->is_halted = true; } @@ -544,10 +572,11 @@ void ts_stack_clear(Stack *self) { } array_clear(&self->heads); array_push(&self->heads, ((StackHead){ - self->base_node, - false, - 0, - NULL + .node = self->base_node, + .last_external_token = NULL, + .depth = 0, + .push_count = 0, + .is_halted = false, })); } @@ -564,15 +593,14 @@ bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) { array_clear(&self->iterators); for (uint32_t i = 0; i < self->heads.size; i++) { + if (ts_stack_is_halted(self, i)) continue; StackHead *head = &self->heads.contents[i]; - if (head->is_halted) - continue; fprintf(f, "node_head_%u [shape=none, label=\"\"]\n", i); fprintf( f, "node_head_%u -> node_%p [label=%u, fontcolor=blue, weight=10000, " - "labeltooltip=\"push_count: %u", - i, head->node, i, head->push_count); + "labeltooltip=\"push_count: %u\ndepth: %u", + i, head->node, i, head->push_count, head->depth); if (head->last_external_token) { const TSExternalTokenState *s = &head->last_external_token->external_token_state; @@ -630,7 +658,7 @@ bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) { fprintf(f, "fontcolor=gray "); if (!link.tree) { - fprintf(f, "color=red, tooltip=\"push_count: %u\"", link.push_count); + fprintf(f, "color=red, tooltip=\"push_count: %u, depth: %u\"", link.push_count, link.depth); } else if (link.tree->symbol == ts_builtin_sym_error) { fprintf(f, "label=\"ERROR\""); } else { diff --git a/src/runtime/stack.h b/src/runtime/stack.h index e64332dc..1eeb5cd0 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -12,8 +12,7 @@ extern "C" { typedef struct Stack Stack; -typedef unsigned int StackVersion; - +typedef unsigned StackVersion; #define STACK_VERSION_NONE ((StackVersion)-1) typedef struct { @@ -28,19 +27,16 @@ typedef struct { StackSliceArray slices; } StackPopResult; +typedef unsigned StackIterateAction; enum { StackIterateNone, - StackIterateStop = 1 << 0, - StackIteratePop = 1 << 1, + StackIterateStop = 1, + StackIteratePop = 2, }; -typedef unsigned int StackIterateAction; - typedef StackIterateAction (*StackIterateCallback)(void *, TSStateId state, - TreeArray *trees, - uint32_t tree_count, - bool is_done, - bool is_pending); + const TreeArray *trees, + uint32_t tree_count); /* * Create a parse stack. @@ -78,10 +74,9 @@ void ts_stack_set_last_external_token(Stack *, StackVersion, Tree *); Length ts_stack_top_position(const Stack *, StackVersion); /* - * Push a tree and state onto the given head of the stack. This could cause - * the version to merge with an existing version. + * Push a tree and state onto the given head of the stack. */ -bool ts_stack_push(Stack *, StackVersion, Tree *, bool, TSStateId); +void ts_stack_push(Stack *, StackVersion, Tree *, bool, TSStateId); /* * Pop the given number of entries from the given version of the stack. This @@ -92,8 +87,7 @@ bool ts_stack_push(Stack *, StackVersion, Tree *, bool, TSStateId); */ StackPopResult ts_stack_pop_count(Stack *, StackVersion, uint32_t count); -StackPopResult ts_stack_iterate(Stack *, StackVersion, StackIterateCallback, - void *); +StackPopResult ts_stack_iterate(Stack *, StackVersion, StackIterateCallback, void *); StackPopResult ts_stack_pop_pending(Stack *, StackVersion); @@ -103,12 +97,18 @@ ErrorStatus ts_stack_error_status(const Stack *, StackVersion); bool ts_stack_merge(Stack *, StackVersion, StackVersion); +bool ts_stack_can_merge(Stack *, StackVersion, StackVersion); + +void ts_stack_force_merge(Stack *, StackVersion, StackVersion); + void ts_stack_halt(Stack *, StackVersion); bool ts_stack_is_halted(Stack *, StackVersion); void ts_stack_renumber_version(Stack *, StackVersion, StackVersion); +void ts_stack_swap_versions(Stack *, StackVersion, StackVersion); + StackVersion ts_stack_copy_version(Stack *, StackVersion); /* diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 92003ff1..b76680e6 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -304,16 +304,6 @@ bool ts_tree_eq(const Tree *self, const Tree *other) { return true; } -bool ts_tree_tokens_eq(const Tree *self, const Tree *other) { - if (self->child_count > 0 || other->child_count > 0) return false; - if (self->symbol != other->symbol) return false; - if (self->padding.bytes != other->padding.bytes) return false; - if (self->size.bytes != other->size.bytes) return false; - if (self->extra != other->extra) return false; - if (!ts_tree_external_token_state_eq(self, other)) return false; - return true; -} - int ts_tree_compare(const Tree *left, const Tree *right) { if (left->symbol < right->symbol) return -1; @@ -553,13 +543,19 @@ void ts_tree_print_dot_graph(const Tree *self, const TSLanguage *language, fprintf(f, "}\n"); } +TSExternalTokenState empty_state = { + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, +}; + bool ts_tree_external_token_state_eq(const Tree *self, const Tree *other) { - return self == other || ( - self && - other && - self->has_external_tokens == other->has_external_tokens && ( - !self->has_external_tokens || - memcmp(&self->external_token_state, &other->external_token_state, sizeof(TSExternalTokenState)) == 0 - ) - ); + const TSExternalTokenState *state1 = &empty_state; + const TSExternalTokenState *state2 = &empty_state; + if (self && self->has_external_tokens) state1 = &self->external_token_state; + if (other && other->has_external_tokens) state2 = &other->external_token_state; + return + state1 == state2 || + memcmp(state1, state2, sizeof(TSExternalTokenState)) == 0; } diff --git a/src/runtime/tree.h b/src/runtime/tree.h index f5f23a9f..e69fcba7 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -79,7 +79,6 @@ Tree *ts_tree_make_error(Length, Length, int32_t); void ts_tree_retain(Tree *tree); void ts_tree_release(Tree *tree); bool ts_tree_eq(const Tree *tree1, const Tree *tree2); -bool ts_tree_tokens_eq(const Tree *, const Tree *); int ts_tree_compare(const Tree *tree1, const Tree *tree2); uint32_t ts_tree_start_column(const Tree *self); diff --git a/test/fixtures/error_corpus/c_errors.txt b/test/fixtures/error_corpus/c_errors.txt index 9057caca..459b4bfd 100644 --- a/test/fixtures/error_corpus/c_errors.txt +++ b/test/fixtures/error_corpus/c_errors.txt @@ -112,10 +112,10 @@ int b() { (function_definition (identifier) (function_declarator (identifier)) (compound_statement - (ERROR (struct_specifier (identifier))) - (expression_statement (number_literal)) - (ERROR (struct_specifier (identifier))) - (expression_statement (number_literal)))) + (struct_specifier (identifier)) + (ERROR (number_literal)) + (struct_specifier (identifier)) + (ERROR (number_literal)))) (function_definition (identifier) (function_declarator (identifier)) diff --git a/test/fixtures/error_corpus/javascript_errors.txt b/test/fixtures/error_corpus/javascript_errors.txt index 5d6a7eb0..02edfb48 100644 --- a/test/fixtures/error_corpus/javascript_errors.txt +++ b/test/fixtures/error_corpus/javascript_errors.txt @@ -30,15 +30,16 @@ h i j k; (program (if_statement - (ERROR (identifier) (identifier)) + (ERROR (identifier)) (identifier) + (ERROR (identifier)) (statement_block (expression_statement - (ERROR (identifier) (jsx_attribute (identifier)) (jsx_attribute (identifier))) - (identifier)))) + (identifier) + (ERROR (jsx_attribute (identifier)) (jsx_attribute (identifier)) (identifier))))) (expression_statement - (ERROR (identifier) (jsx_attribute (identifier)) (jsx_attribute (identifier))) - (identifier))) + (identifier) + (ERROR (jsx_attribute (identifier)) (jsx_attribute (identifier)) (identifier)))) =================================================== one invalid subtree right after the viable prefix @@ -52,8 +53,8 @@ if ({a: 'b'} {c: 'd'}) { (program (if_statement - (ERROR (object (pair (identifier) (string)))) (object (pair (identifier) (string))) + (ERROR (statement_block (labeled_statement (identifier) (expression_statement (string))))) (statement_block (expression_statement (assignment (identifier) diff --git a/test/runtime/stack_test.cc b/test/runtime/stack_test.cc index 14bcbc7c..03bf7091 100644 --- a/test/runtime/stack_test.cc +++ b/test/runtime/stack_test.cc @@ -54,11 +54,12 @@ vector get_stack_entries(Stack *stack, StackVersion version) { ts_stack_iterate( stack, version, - [](void *payload, TSStateId state, TreeArray *trees, uint32_t tree_count, bool is_done, bool is_pending) -> StackIterateAction { + [](void *payload, TSStateId state, const TreeArray *trees, uint32_t tree_count) -> StackIterateAction { auto entries = static_cast *>(payload); StackEntry entry = {state, tree_count}; - if (find(entries->begin(), entries->end(), entry) == entries->end()) + if (find(entries->begin(), entries->end(), entry) == entries->end()) { entries->push_back(entry); + } return StackIterateNone; }, &result); return result; @@ -524,8 +525,10 @@ describe("Stack", [&]() { describe("setting external token state", [&]() { before_each([&]() { - trees[1]->external_token_state[0] = 'a'; - trees[2]->external_token_state[0] = 'b'; + trees[1]->has_external_tokens = true; + trees[2]->has_external_tokens = true; + memset(&trees[1]->external_token_state, 0, sizeof(TSExternalTokenState)); + memset(&trees[2]->external_token_state, 0, sizeof(TSExternalTokenState)); }); it("allows the state to be retrieved", [&]() { @@ -535,19 +538,49 @@ describe("Stack", [&]() { AssertThat(ts_stack_last_external_token(stack, 0), Equals(trees[1])); ts_stack_copy_version(stack, 0); - AssertThat(ts_stack_last_external_token(stack, 0), Equals(trees[1])); + AssertThat(ts_stack_last_external_token(stack, 1), Equals(trees[1])); + + ts_stack_set_last_external_token(stack, 0, trees[2]); + AssertThat(ts_stack_last_external_token(stack, 0), Equals(trees[2])); }); it("does not merge stack versions with different external token states", [&]() { + trees[1]->external_token_state[5] = 'a'; + trees[2]->external_token_state[5] = 'b'; + ts_stack_copy_version(stack, 0); ts_stack_push(stack, 0, trees[0], false, 5); ts_stack_push(stack, 1, trees[0], false, 5); ts_stack_set_last_external_token(stack, 0, trees[1]); - ts_stack_set_last_external_token(stack, 0, trees[1]); + ts_stack_set_last_external_token(stack, 1, trees[2]); AssertThat(ts_stack_merge(stack, 0, 1), IsFalse()); }); + + it("merges stack versions with identical external token states", [&]() { + trees[1]->external_token_state[5] = 'a'; + trees[2]->external_token_state[5] = 'a'; + + ts_stack_copy_version(stack, 0); + ts_stack_push(stack, 0, trees[0], false, 5); + ts_stack_push(stack, 1, trees[0], false, 5); + + ts_stack_set_last_external_token(stack, 0, trees[1]); + ts_stack_set_last_external_token(stack, 1, trees[2]); + + AssertThat(ts_stack_merge(stack, 0, 1), IsTrue()); + }); + + it("does not distinguish between an *empty* external token state and *no* external token state", [&]() { + ts_stack_copy_version(stack, 0); + ts_stack_push(stack, 0, trees[0], false, 5); + ts_stack_push(stack, 1, trees[0], false, 5); + + ts_stack_set_last_external_token(stack, 0, trees[1]); + + AssertThat(ts_stack_merge(stack, 0, 1), IsTrue()); + }); }); });