From b29d0f622fd09dd123fb734e85be9fd823f2fbcb Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 17 Sep 2018 13:12:13 -0700 Subject: [PATCH] Cram terminal subtree data into a 64-bit integer when possible --- project.gyp | 2 +- src/runtime/alloc.h | 2 +- src/runtime/get_changed_ranges.c | 67 +-- src/runtime/node.c | 126 ++--- src/runtime/parser.c | 402 ++++++++-------- src/runtime/reusable_node.h | 26 +- src/runtime/stack.c | 140 +++--- src/runtime/stack.h | 6 +- src/runtime/subtree.c | 800 +++++++++++++++++-------------- src/runtime/subtree.h | 229 +++++++-- src/runtime/tree.c | 8 +- src/runtime/tree.h | 4 +- src/runtime/tree_cursor.c | 46 +- test/helpers/point_helpers.cc | 8 + test/helpers/point_helpers.h | 4 + test/helpers/tree_helpers.cc | 17 +- test/helpers/tree_helpers.h | 6 +- test/runtime/parser_test.cc | 2 +- test/runtime/stack_test.cc | 44 +- test/runtime/subtree_test.cc | 324 ++++++------- test/test_helper.h | 2 +- 21 files changed, 1258 insertions(+), 1007 deletions(-) diff --git a/project.gyp b/project.gyp index 77002a65..38f6e551 100644 --- a/project.gyp +++ b/project.gyp @@ -127,7 +127,7 @@ }, }, 'Test': { - 'defines': ['TREE_SITTER_WRAP_MALLOC=true'], + 'defines': ['TREE_SITTER_TEST=true'], 'cflags': [ '-g' ], 'ldflags': [ '-g' ], 'xcode_settings': { diff --git a/src/runtime/alloc.h b/src/runtime/alloc.h index 9561fa1f..6f3fa347 100644 --- a/src/runtime/alloc.h +++ b/src/runtime/alloc.h @@ -9,7 +9,7 @@ extern "C" { #include #include -#if defined(TREE_SITTER_WRAP_MALLOC) +#if defined(TREE_SITTER_TEST) void *ts_record_malloc(size_t); void *ts_record_calloc(size_t, size_t); diff --git a/src/runtime/get_changed_ranges.c b/src/runtime/get_changed_ranges.c index 9054e65a..40e61c47 100644 --- a/src/runtime/get_changed_ranges.c +++ b/src/runtime/get_changed_ranges.c @@ -57,32 +57,35 @@ Length iterator_start_position(Iterator *self) { if (self->in_padding) { return entry.position; } else { - return length_add(entry.position, entry.subtree->padding); + return length_add(entry.position, ts_subtree_padding(*entry.subtree)); } } Length iterator_end_position(Iterator *self) { TreeCursorEntry entry = *array_back(&self->cursor.stack); - Length result = length_add(entry.position, entry.subtree->padding); + Length result = length_add(entry.position, ts_subtree_padding(*entry.subtree)); if (self->in_padding) { return result; } else { - return length_add(result, entry.subtree->size); + return length_add(result, ts_subtree_size(*entry.subtree)); } } static bool iterator_tree_is_visible(const Iterator *self) { TreeCursorEntry entry = *array_back(&self->cursor.stack); - if (entry.subtree->visible) return true; + if (ts_subtree_visible(*entry.subtree)) return true; if (self->cursor.stack.size > 1) { - const Subtree *parent = self->cursor.stack.contents[self->cursor.stack.size - 2].subtree; - const TSSymbol *alias_sequence = ts_language_alias_sequence(self->language, parent->alias_sequence_id); + Subtree parent = *self->cursor.stack.contents[self->cursor.stack.size - 2].subtree; + const TSSymbol *alias_sequence = ts_language_alias_sequence( + self->language, + parent.ptr->alias_sequence_id + ); return alias_sequence && alias_sequence[entry.structural_child_index] != 0; } return false; } -static void iterator_get_visible_state(const Iterator *self, const Subtree **tree, +static void iterator_get_visible_state(const Iterator *self, Subtree *tree, TSSymbol *alias_symbol, uint32_t *start_byte) { uint32_t i = self->cursor.stack.size - 1; @@ -98,15 +101,15 @@ static void iterator_get_visible_state(const Iterator *self, const Subtree **tre const Subtree *parent = self->cursor.stack.contents[i - 1].subtree; const TSSymbol *alias_sequence = ts_language_alias_sequence( self->language, - parent->alias_sequence_id + parent->ptr->alias_sequence_id ); if (alias_sequence) { *alias_symbol = alias_sequence[entry.structural_child_index]; } } - if (entry.subtree->visible || *alias_symbol) { - *tree = entry.subtree; + if (ts_subtree_visible(*entry.subtree) || *alias_symbol) { + *tree = *entry.subtree; *start_byte = entry.position.bytes; break; } @@ -129,10 +132,10 @@ static bool iterator_descend(Iterator *self, uint32_t goal_position) { TreeCursorEntry entry = *array_back(&self->cursor.stack); Length position = entry.position; uint32_t structural_child_index = 0; - for (uint32_t i = 0; i < entry.subtree->child_count; i++) { - const Subtree *child = entry.subtree->children[i]; - Length child_left = length_add(position, child->padding); - Length child_right = length_add(child_left, child->size); + for (uint32_t i = 0, n = ts_subtree_child_count(*entry.subtree); i < n; i++) { + const Subtree *child = &entry.subtree->ptr->children[i]; + Length child_left = length_add(position, ts_subtree_padding(*child)); + Length child_right = length_add(child_left, ts_subtree_size(*child)); if (child_right.bytes > goal_position) { array_push(&self->cursor.stack, ((TreeCursorEntry){ @@ -156,7 +159,7 @@ static bool iterator_descend(Iterator *self, uint32_t goal_position) { } position = child_right; - if (!child->extra) structural_child_index++; + if (!ts_subtree_extra(*child)) structural_child_index++; } } while (did_descend); @@ -181,11 +184,11 @@ static void iterator_advance(Iterator *self) { const Subtree *parent = array_back(&self->cursor.stack)->subtree; uint32_t child_index = entry.child_index + 1; - if (parent->child_count > child_index) { - Length position = length_add(entry.position, ts_subtree_total_size(entry.subtree)); + if (ts_subtree_child_count(*parent) > child_index) { + Length position = length_add(entry.position, ts_subtree_total_size(*entry.subtree)); uint32_t structural_child_index = entry.structural_child_index; - if (!entry.subtree->extra) structural_child_index++; - const Subtree *next_child = parent->children[child_index]; + if (!ts_subtree_extra(*entry.subtree)) structural_child_index++; + const Subtree *next_child = &parent->ptr->children[child_index]; array_push(&self->cursor.stack, ((TreeCursorEntry){ .subtree = next_child, @@ -195,7 +198,7 @@ static void iterator_advance(Iterator *self) { })); if (iterator_tree_is_visible(self)) { - if (next_child->padding.bytes > 0) { + if (ts_subtree_padding(*next_child).bytes > 0) { self->in_padding = true; } else { self->visible_depth++; @@ -215,30 +218,30 @@ typedef enum { } IteratorComparison; IteratorComparison iterator_compare(const Iterator *old_iter, const Iterator *new_iter) { - const Subtree *old_tree = NULL, *new_tree = NULL; + Subtree old_tree = {.ptr = NULL}, new_tree = {.ptr = NULL}; uint32_t old_start = 0, new_start = 0; TSSymbol old_alias_symbol = 0, new_alias_symbol = 0; iterator_get_visible_state(old_iter, &old_tree, &old_alias_symbol, &old_start); iterator_get_visible_state(new_iter, &new_tree, &new_alias_symbol, &new_start); - if (!old_tree && !new_tree) return IteratorMatches; - if (!old_tree || !new_tree) return IteratorDiffers; + if (!old_tree.ptr && !new_tree.ptr) return IteratorMatches; + if (!old_tree.ptr || !new_tree.ptr) return IteratorDiffers; if (old_alias_symbol == new_alias_symbol) { if (old_start == new_start) { - if (!old_tree->has_changes && - old_tree->symbol == new_tree->symbol && - old_tree->symbol != ts_builtin_sym_error && - old_tree->size.bytes == new_tree->size.bytes && - old_tree->parse_state != TS_TREE_STATE_NONE && - new_tree->parse_state != TS_TREE_STATE_NONE && - (old_tree->parse_state == ERROR_STATE) == - (new_tree->parse_state == ERROR_STATE)) { + if (!ts_subtree_has_changes(old_tree) && + ts_subtree_symbol(old_tree) == ts_subtree_symbol(new_tree) && + ts_subtree_symbol(old_tree) != ts_builtin_sym_error && + ts_subtree_size(old_tree).bytes == ts_subtree_size(new_tree).bytes && + ts_subtree_parse_state(old_tree) != TS_TREE_STATE_NONE && + ts_subtree_parse_state(new_tree) != TS_TREE_STATE_NONE && + (ts_subtree_parse_state(old_tree) == ERROR_STATE) == + (ts_subtree_parse_state(new_tree) == ERROR_STATE)) { return IteratorMatches; } } - if (old_tree->symbol == new_tree->symbol) { + if (ts_subtree_symbol(old_tree) == ts_subtree_symbol(new_tree)) { return IteratorMayDiffer; } } diff --git a/src/runtime/node.c b/src/runtime/node.c index 9ae5dad2..5a528f03 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -4,7 +4,7 @@ #include "runtime/language.h" typedef struct { - const Subtree *parent; + Subtree parent; const TSTree *tree; Length position; uint32_t child_index; @@ -40,17 +40,20 @@ static inline uint32_t ts_node__alias(const TSNode *self) { return self->context[3]; } -static inline const Subtree *ts_node__subtree(TSNode self) { - return self.id; +static inline Subtree ts_node__subtree(TSNode self) { + return *(const Subtree *)self.id; } // ChildIterator static inline ChildIterator ts_node_iterate_children(const TSNode *node) { - const Subtree *subtree = ts_node__subtree(*node); + Subtree subtree = ts_node__subtree(*node); + if (ts_subtree_child_count(subtree) == 0) { + return (ChildIterator) {NULL_SUBTREE, node->tree, length_zero(), 0, 0, NULL}; + } const TSSymbol *alias_sequence = ts_language_alias_sequence( node->tree->language, - subtree->alias_sequence_id + subtree.ptr->alias_sequence_id ); return (ChildIterator) { .tree = node->tree, @@ -63,17 +66,17 @@ static inline ChildIterator ts_node_iterate_children(const TSNode *node) { } static inline bool ts_node_child_iterator_next(ChildIterator *self, TSNode *result) { - if (self->child_index == self->parent->child_count) return false; - const Subtree *child = self->parent->children[self->child_index]; + if (!self->parent.ptr || self->child_index == self->parent.ptr->child_count) return false; + const Subtree *child = &self->parent.ptr->children[self->child_index]; TSSymbol alias_symbol = 0; - if (!child->extra) { + if (!ts_subtree_extra(*child)) { if (self->alias_sequence) { alias_symbol = self->alias_sequence[self->structural_child_index]; } self->structural_child_index++; } if (self->child_index > 0) { - self->position = length_add(self->position, child->padding); + self->position = length_add(self->position, ts_subtree_padding(*child)); } *result = ts_node_new( self->tree, @@ -81,7 +84,7 @@ static inline bool ts_node_child_iterator_next(ChildIterator *self, TSNode *resu self->position, alias_symbol ); - self->position = length_add(self->position, child->size); + self->position = length_add(self->position, ts_subtree_size(*child)); self->child_index++; return true; } @@ -89,30 +92,25 @@ static inline bool ts_node_child_iterator_next(ChildIterator *self, TSNode *resu // TSNode - private static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { - const Subtree *tree = ts_node__subtree(self); + Subtree tree = ts_node__subtree(self); if (include_anonymous) { - return tree->visible || ts_node__alias(&self); + return ts_subtree_visible(tree) || ts_node__alias(&self); } else { - return ( - (tree->visible && tree->named) || - ( - ts_node__alias(&self) && - ts_language_symbol_metadata( - self.tree->language, - ts_node__alias(&self) - ).named - ) - ); + return + (ts_subtree_visible(tree) && + ts_subtree_named(tree)) || + (ts_node__alias(&self) && + ts_language_symbol_metadata(self.tree->language, ts_node__alias(&self)).named); } } static inline uint32_t ts_node__relevant_child_count(TSNode self, bool include_anonymous) { - const Subtree *tree = ts_node__subtree(self); - if (tree->child_count > 0) { + Subtree tree = ts_node__subtree(self); + if (ts_subtree_child_count(tree) > 0) { if (include_anonymous) { - return tree->visible_child_count; + return tree.ptr->visible_child_count; } else { - return tree->named_child_count; + return tree.ptr->named_child_count; } } else { return 0; @@ -153,25 +151,27 @@ static inline TSNode ts_node__child(TSNode self, uint32_t child_index, bool incl return ts_node__null(); } -static bool ts_subtree_has_trailing_empty_descendant(const Subtree *self, const Subtree *other) { - for (unsigned i = self->child_count - 1; i + 1 > 0; i--) { - const Subtree *child = self->children[i]; - if (child->size.bytes > 0 || child->padding.bytes > 0) break; - if (child == other || ts_subtree_has_trailing_empty_descendant(child, other)) return true; +static bool ts_subtree_has_trailing_empty_descendant(Subtree self, Subtree other) { + for (unsigned i = ts_subtree_child_count(self) - 1; i + 1 > 0; i--) { + Subtree child = self.ptr->children[i]; + if (ts_subtree_total_bytes(child) > 0) break; + if (child.ptr == other.ptr || ts_subtree_has_trailing_empty_descendant(child, other)) { + return true; + } } return false; } static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) { - const Subtree *self_subtree = ts_node__subtree(self); - bool self_is_empty = self_subtree->size.bytes == 0 && self_subtree->padding.bytes == 0; + Subtree self_subtree = ts_node__subtree(self); + bool self_is_empty = ts_subtree_total_bytes(self_subtree) == 0; uint32_t target_end_byte = ts_node_end_byte(self); TSNode node = ts_node_parent(self); TSNode earlier_node = ts_node__null(); bool earlier_node_is_relevant = false; - while (ts_node__subtree(node)) { + while (!ts_node_is_null(node)) { TSNode earlier_child = ts_node__null(); bool earlier_child_is_relevant = false; bool found_child_containing_target = false; @@ -180,11 +180,14 @@ static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) ChildIterator iterator = ts_node_iterate_children(&node); while (ts_node_child_iterator_next(&iterator, &child)) { if (child.id == self.id) break; + if (iterator.position.bytes > target_end_byte) { + found_child_containing_target = true; + break; + } - if (iterator.position.bytes > target_end_byte || ( - iterator.position.bytes == target_end_byte && ( - !self_is_empty || - ts_subtree_has_trailing_empty_descendant(ts_node__subtree(child), self_subtree)))) { + if (iterator.position.bytes == target_end_byte && + (!self_is_empty || + ts_subtree_has_trailing_empty_descendant(ts_node__subtree(child), self_subtree))) { found_child_containing_target = true; break; } @@ -225,7 +228,7 @@ static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) TSNode later_node = ts_node__null(); bool later_node_is_relevant = false; - while (ts_node__subtree(node)) { + while (!ts_node_is_null(node)) { TSNode later_child = ts_node__null(); bool later_child_is_relevant = false; TSNode child_containing_target = ts_node__null(); @@ -235,7 +238,7 @@ static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) while (ts_node_child_iterator_next(&iterator, &child)) { if (iterator.position.bytes < target_end_byte) continue; if (ts_node_start_byte(child) <= ts_node_start_byte(self)) { - if (ts_node__subtree(child) != ts_node__subtree(self)) { + if (ts_node__subtree(child).ptr != ts_node__subtree(self).ptr) { child_containing_target = child; } } else if (ts_node__is_relevant(child, include_anonymous)) { @@ -360,16 +363,17 @@ static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint mi // TSNode - public uint32_t ts_node_end_byte(TSNode self) { - return ts_node_start_byte(self) + ts_node__subtree(self)->size.bytes; + return ts_node_start_byte(self) + ts_subtree_size(ts_node__subtree(self)).bytes; } TSPoint ts_node_end_point(TSNode self) { - return point_add(ts_node_start_point(self), ts_node__subtree(self)->size.extent); + return point_add(ts_node_start_point(self), ts_subtree_size(ts_node__subtree(self)).extent); } TSSymbol ts_node_symbol(TSNode self) { - const Subtree *tree = ts_node__subtree(self); - return ts_node__alias(&self) ? ts_node__alias(&self) : tree->symbol; + return ts_node__alias(&self) + ? ts_node__alias(&self) + : ts_subtree_symbol(ts_node__subtree(self)); } const char *ts_node_type(TSNode self) { @@ -385,27 +389,25 @@ bool ts_node_eq(TSNode self, TSNode other) { } bool ts_node_is_null(TSNode self) { - return ts_node__subtree(self) == NULL; + return self.id == 0; } bool ts_node_is_named(TSNode self) { - const Subtree *tree = ts_node__subtree(self); return ts_node__alias(&self) ? ts_language_symbol_metadata(self.tree->language, ts_node__alias(&self)).named - : tree->named; + : ts_subtree_named(ts_node__subtree(self)); } bool ts_node_is_missing(TSNode self) { - const Subtree *tree = ts_node__subtree(self); - return tree->is_missing; + return ts_subtree_missing(ts_node__subtree(self)); } bool ts_node_has_changes(TSNode self) { - return ts_node__subtree(self)->has_changes; + return ts_subtree_has_changes(ts_node__subtree(self)); } bool ts_node_has_error(TSNode self) { - return ts_node__subtree(self)->error_cost > 0; + return ts_subtree_error_cost(ts_node__subtree(self)) > 0; } TSNode ts_node_parent(TSNode self) { @@ -414,7 +416,7 @@ TSNode ts_node_parent(TSNode self) { node = ts_tree_root_node(self.tree); uint32_t end_byte = ts_node_end_byte(self); - if (ts_node__subtree(node) == ts_node__subtree(self)) return ts_node__null(); + if (node.id == self.id) return ts_node__null(); TSNode last_visible_node = node; bool did_descend = true; @@ -426,7 +428,7 @@ TSNode ts_node_parent(TSNode self) { while (ts_node_child_iterator_next(&iterator, &child)) { if ( ts_node_start_byte(child) > ts_node_start_byte(self) || - ts_node__subtree(child) == ts_node__subtree(self) + child.id == self.id ) break; if (iterator.position.bytes >= end_byte) { node = child; @@ -452,18 +454,18 @@ TSNode ts_node_named_child(TSNode self, uint32_t child_index) { } uint32_t ts_node_child_count(TSNode self) { - const Subtree *tree = ts_node__subtree(self); - if (tree->child_count > 0) { - return tree->visible_child_count; + Subtree tree = ts_node__subtree(self); + if (ts_subtree_child_count(tree) > 0) { + return tree.ptr->visible_child_count; } else { return 0; } } uint32_t ts_node_named_child_count(TSNode self) { - const Subtree *tree = ts_node__subtree(self); - if (tree->child_count > 0) { - return tree->named_child_count; + Subtree tree = ts_node__subtree(self); + if (ts_subtree_child_count(tree) > 0) { + return tree.ptr->named_child_count; } else { return 0; } @@ -497,8 +499,7 @@ TSNode ts_node_descendant_for_byte_range(TSNode self, uint32_t min, uint32_t max return ts_node__descendant_for_byte_range(self, min, max, true); } -TSNode ts_node_named_descendant_for_byte_range(TSNode self, uint32_t min, - uint32_t max) { +TSNode ts_node_named_descendant_for_byte_range(TSNode self, uint32_t min, uint32_t max) { return ts_node__descendant_for_byte_range(self, min, max, false); } @@ -506,8 +507,7 @@ TSNode ts_node_descendant_for_point_range(TSNode self, TSPoint min, TSPoint max) return ts_node__descendant_for_point_range(self, min, max, true); } -TSNode ts_node_named_descendant_for_point_range(TSNode self, TSPoint min, - TSPoint max) { +TSNode ts_node_named_descendant_for_point_range(TSNode self, TSPoint min, TSPoint max) { return ts_node__descendant_for_point_range(self, min, max, false); } diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 92077144..37487401 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -35,14 +35,16 @@ #define SYM_NAME(symbol) ts_language_symbol_name(self->language, symbol) +#define TREE_NAME(tree) SYM_NAME(ts_subtree_symbol(tree)) + static const unsigned MAX_VERSION_COUNT = 6; static const unsigned MAX_VERSION_COUNT_OVERFLOW = 4; static const unsigned MAX_SUMMARY_DEPTH = 16; static const unsigned MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE; typedef struct { - const Subtree *token; - const Subtree *last_external_token; + Subtree token; + Subtree last_external_token; uint32_t byte_index; } TokenCache; @@ -52,8 +54,9 @@ struct TSParser { SubtreePool tree_pool; const TSLanguage *language; ReduceActionSet reduce_actions; - const Subtree *finished_tree; - Subtree scratch_tree; + Subtree finished_tree; + SubtreeHeapData scratch_tree_data; + MutableSubtree scratch_tree; TokenCache token_cache; ReusableNode reusable_node; void *external_scanner_payload; @@ -62,7 +65,7 @@ struct TSParser { size_t operation_limit; volatile bool enabled; bool halt_on_error; - const Subtree *old_tree; + Subtree old_tree; }; typedef struct { @@ -132,16 +135,16 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion versi for (uint32_t i = 0; i < pop.size; i++) { StackSlice slice = pop.contents[i]; TSStateId state = ts_stack_state(self->stack, slice.version); - const Subtree *parent = *array_front(&slice.subtrees); + Subtree parent = *array_front(&slice.subtrees); - for (uint32_t j = 0; j < parent->child_count; j++) { - const Subtree *child = parent->children[j]; - pending = child->child_count > 0; + for (uint32_t j = 0, n = ts_subtree_child_count(parent); j < n; j++) { + Subtree child = parent.ptr->children[j]; + pending = ts_subtree_child_count(child) > 0; - if (child->symbol == ts_builtin_sym_error) { + if (ts_subtree_is_error(child)) { state = ERROR_STATE; - } else if (!child->extra) { - state = ts_language_next_state(self->language, state, child->symbol); + } else if (!ts_subtree_extra(child)) { + state = ts_language_next_state(self->language, state, ts_subtree_symbol(child)); } ts_subtree_retain(child); @@ -149,14 +152,14 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion versi } for (uint32_t j = 1; j < slice.subtrees.size; j++) { - const Subtree *tree = slice.subtrees.contents[j]; + Subtree tree = slice.subtrees.contents[j]; ts_stack_push(self->stack, slice.version, tree, false, state); } ts_subtree_release(&self->tree_pool, parent); array_delete(&slice.subtrees); - LOG("breakdown_top_of_stack tree:%s", SYM_NAME(parent->symbol)); + LOG("breakdown_top_of_stack tree:%s", TREE_NAME(parent)); LOG_STACK(); } } while (pending); @@ -164,12 +167,12 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion versi return did_break_down; } -static void ts_parser__breakdown_lookahead(TSParser *self, const Subtree **lookahead, +static void ts_parser__breakdown_lookahead(TSParser *self, Subtree *lookahead, TSStateId state, ReusableNode *reusable_node) { bool did_descend = false; - const Subtree *tree = reusable_node_tree(reusable_node); - while (tree->child_count > 0 && tree->parse_state != state) { - LOG("state_mismatch sym:%s", SYM_NAME(tree->symbol)); + Subtree tree = reusable_node_tree(reusable_node); + while (ts_subtree_child_count(tree) > 0 && ts_subtree_parse_state(tree) != state) { + LOG("state_mismatch sym:%s", TREE_NAME(tree)); reusable_node_descend(reusable_node); tree = reusable_node_tree(reusable_node); did_descend = true; @@ -233,8 +236,10 @@ static ErrorStatus ts_parser__version_status(TSParser *self, StackVersion versio } static bool ts_parser__better_version_exists(TSParser *self, StackVersion version, - bool is_in_error, unsigned cost) { - if (self->finished_tree && self->finished_tree->error_cost <= cost) return true; + bool is_in_error, unsigned cost) { + if (self->finished_tree.ptr && ts_subtree_error_cost(self->finished_tree) <= cost) { + return true; + } Length position = ts_stack_position(self->stack, version); ErrorStatus status = { @@ -262,50 +267,41 @@ static bool ts_parser__better_version_exists(TSParser *self, StackVersion versio return false; } -static void ts_parser__restore_external_scanner(TSParser *self, const Subtree *external_token) { - if (external_token) { +static void ts_parser__restore_external_scanner(TSParser *self, Subtree external_token) { + if (external_token.ptr) { self->language->external_scanner.deserialize( self->external_scanner_payload, - ts_external_scanner_state_data(&external_token->external_scanner_state), - external_token->external_scanner_state.length + ts_external_scanner_state_data(&external_token.ptr->external_scanner_state), + external_token.ptr->external_scanner_state.length ); } else { self->language->external_scanner.deserialize(self->external_scanner_payload, NULL, 0); } } -static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, const Subtree *tree, +static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, Subtree tree, TableEntry *table_entry) { TSLexMode current_lex_mode = self->language->lex_modes[state]; - - TSStateId leaf_state; - TSSymbol leaf_symbol; - if (tree->child_count > 0) { - leaf_state = tree->first_leaf.parse_state; - leaf_symbol = tree->first_leaf.symbol; - } else { - leaf_state = tree->parse_state; - leaf_symbol = tree->symbol; - } - + TSSymbol leaf_symbol = ts_subtree_leaf_symbol(tree); + TSStateId leaf_state = ts_subtree_leaf_parse_state(tree); TSLexMode leaf_lex_mode = self->language->lex_modes[leaf_state]; // If the token was created in a state with the same set of lookaheads, it is reusable. if (memcmp(&leaf_lex_mode, ¤t_lex_mode, sizeof(TSLexMode)) == 0 && (leaf_symbol != self->language->keyword_capture_token || - (!tree->is_keyword && tree->parse_state == state))) return true; + (!ts_subtree_is_keyword(tree) && ts_subtree_parse_state(tree) == state))) return true; // Empty tokens are not reusable in states with different lookaheads. - if (tree->size.bytes == 0 && tree->symbol != ts_builtin_sym_end) return false; + if (ts_subtree_size(tree).bytes == 0 && leaf_symbol != ts_builtin_sym_end) return false; // If the current state allows external tokens or other tokens that conflict with this // token, this token is not reusable. return current_lex_mode.external_lex_state == 0 && table_entry->is_reusable; } -static const Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId parse_state) { +static Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId parse_state) { Length start_position = ts_stack_position(self->stack, version); - const Subtree *external_token = ts_stack_last_external_token(self->stack, version); + Subtree external_token = ts_stack_last_external_token(self->stack, version); TSLexMode lex_mode = self->language->lex_modes[parse_state]; const bool *valid_external_tokens = ts_language_enabled_external_tokens( self->language, @@ -404,13 +400,19 @@ static const Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSSta uint32_t bytes_scanned = last_byte_scanned - start_position.bytes + 1; - Subtree *result; + Subtree result; if (skipped_error) { Length padding = length_sub(error_start_position, start_position); Length size = length_sub(error_end_position, error_start_position); - result = ts_subtree_new_error(&self->tree_pool, size, padding, first_error_character, self->language); - result->parse_state = parse_state; - result->bytes_scanned = bytes_scanned; + result = ts_subtree_new_error( + &self->tree_pool, + first_error_character, + padding, + size, + bytes_scanned, + parse_state, + self->language + ); } else { if (self->lexer.token_end_position.bytes < self->lexer.token_start_position.bytes) { self->lexer.token_start_position = self->lexer.token_end_position; @@ -455,85 +457,87 @@ static const Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSSta self->lexer.debug_buffer ); ts_external_scanner_state_init( - &result->external_scanner_state, + &((SubtreeHeapData *)result.ptr)->external_scanner_state, self->lexer.debug_buffer, length ); } } - LOG("lexed_lookahead sym:%s, size:%u", SYM_NAME(result->symbol), result->size.bytes); + LOG( + "lexed_lookahead sym:%s, size:%u", + SYM_NAME(ts_subtree_symbol(result)), + ts_subtree_size(result).bytes + ); return result; } -static const Subtree *ts_parser__get_cached_token(TSParser *self, TSStateId state, - size_t position, - const Subtree *last_external_token, - TableEntry *table_entry) { +static Subtree ts_parser__get_cached_token(TSParser *self, TSStateId state, + size_t position, Subtree last_external_token, + TableEntry *table_entry) { TokenCache *cache = &self->token_cache; if ( - cache->token && cache->byte_index == position && + cache->token.ptr && cache->byte_index == position && ts_subtree_external_scanner_state_eq(cache->last_external_token, last_external_token) ) { - ts_language_table_entry(self->language, state, cache->token->symbol, table_entry); + ts_language_table_entry(self->language, state, ts_subtree_symbol(cache->token), table_entry); if (ts_parser__can_reuse_first_leaf(self, state, cache->token, table_entry)) { ts_subtree_retain(cache->token); return cache->token; } } - return NULL; + return NULL_SUBTREE; } static void ts_parser__set_cached_token(TSParser *self, size_t byte_index, - const Subtree *last_external_token, - const Subtree *token) { + Subtree last_external_token, + Subtree token) { TokenCache *cache = &self->token_cache; - if (token) ts_subtree_retain(token); - if (last_external_token) ts_subtree_retain(last_external_token); - if (cache->token) ts_subtree_release(&self->tree_pool, cache->token); - if (cache->last_external_token) ts_subtree_release(&self->tree_pool, cache->last_external_token); + if (token.ptr) ts_subtree_retain(token); + if (last_external_token.ptr) ts_subtree_retain(last_external_token); + if (cache->token.ptr) ts_subtree_release(&self->tree_pool, cache->token); + if (cache->last_external_token.ptr) ts_subtree_release(&self->tree_pool, cache->last_external_token); cache->token = token; cache->byte_index = byte_index; cache->last_external_token = last_external_token; } -static const Subtree *ts_parser__reuse_node(TSParser *self, StackVersion version, - TSStateId *state, uint32_t position, - const Subtree *last_external_token, - TableEntry *table_entry) { - const Subtree *result; - while ((result = reusable_node_tree(&self->reusable_node))) { +static Subtree ts_parser__reuse_node(TSParser *self, StackVersion version, + TSStateId *state, uint32_t position, + Subtree last_external_token, TableEntry *table_entry) { + Subtree result; + while ((result = reusable_node_tree(&self->reusable_node)).ptr) { uint32_t byte_offset = reusable_node_byte_offset(&self->reusable_node); if (byte_offset > position) { - LOG("before_reusable_node symbol:%s", SYM_NAME(result->symbol)); + LOG("before_reusable_node symbol:%s", TREE_NAME(result)); break; } if (byte_offset < position) { - LOG("past_reusable_node symbol:%s", SYM_NAME(result->symbol)); + LOG("past_reusable_node symbol:%s", TREE_NAME(result)); reusable_node_advance(&self->reusable_node); continue; } if (!ts_subtree_external_scanner_state_eq(self->reusable_node.last_external_token, last_external_token)) { - LOG("reusable_node_has_different_external_scanner_state symbol:%s", SYM_NAME(result->symbol)); + LOG("reusable_node_has_different_external_scanner_state symbol:%s", TREE_NAME(result)); reusable_node_advance(&self->reusable_node); continue; } const char *reason = NULL; - if (result->has_changes) { + if (ts_subtree_has_changes(result)) { reason = "has_changes"; - } else if (result->symbol == ts_builtin_sym_error) { + } else if (ts_subtree_is_error(result)) { reason = "is_error"; - } else if (result->is_missing) { + } else if (ts_subtree_missing(result)) { reason = "is_missing"; - } else if (result->fragile_left || result->fragile_right) { + } else if (ts_subtree_is_fragile(result)) { reason = "is_fragile"; } if (reason) { - LOG("cant_reuse_node_%s tree:%s", reason, SYM_NAME(result->symbol)); + LOG("cant_reuse_node_%s tree:%s", reason, TREE_NAME(result)); if (!reusable_node_descend(&self->reusable_node)) { reusable_node_advance(&self->reusable_node); ts_parser__breakdown_top_of_stack(self, version); @@ -547,97 +551,92 @@ static const Subtree *ts_parser__reuse_node(TSParser *self, StackVersion version if (!ts_parser__can_reuse_first_leaf(self, *state, result, table_entry)) { LOG( "cant_reuse_node symbol:%s, first_leaf_symbol:%s", - SYM_NAME(result->symbol), + TREE_NAME(result), SYM_NAME(leaf_symbol) ); reusable_node_advance_past_leaf(&self->reusable_node); break; } - LOG("reuse_node symbol:%s", SYM_NAME(result->symbol)); + LOG("reuse_node symbol:%s", TREE_NAME(result)); ts_subtree_retain(result); return result; } - return NULL; + return NULL_SUBTREE; } -static bool ts_parser__select_tree(TSParser *self, const Subtree *left, const Subtree *right) { - if (!left) return true; - if (!right) return false; +static bool ts_parser__select_tree(TSParser *self, Subtree left, Subtree right) { + if (!left.ptr) return true; + if (!right.ptr) return false; - if (right->error_cost < left->error_cost) { - LOG("select_smaller_error symbol:%s, over_symbol:%s", - SYM_NAME(right->symbol), SYM_NAME(left->symbol)); + if (ts_subtree_error_cost(right) < ts_subtree_error_cost(left)) { + LOG("select_smaller_error symbol:%s, over_symbol:%s", TREE_NAME(right), TREE_NAME(left)); return true; } - if (left->error_cost < right->error_cost) { - LOG("select_smaller_error symbol:%s, over_symbol:%s", - SYM_NAME(left->symbol), SYM_NAME(right->symbol)); + if (ts_subtree_error_cost(left) < ts_subtree_error_cost(right)) { + LOG("select_smaller_error symbol:%s, over_symbol:%s", TREE_NAME(left), TREE_NAME(right)); return false; } - if (right->dynamic_precedence > left->dynamic_precedence) { + if (ts_subtree_dynamic_precedence(right) > ts_subtree_dynamic_precedence(left)) { LOG("select_higher_precedence symbol:%s, prec:%u, over_symbol:%s, other_prec:%u", - SYM_NAME(right->symbol), right->dynamic_precedence, SYM_NAME(left->symbol), - left->dynamic_precedence); + TREE_NAME(right), ts_subtree_dynamic_precedence(right), TREE_NAME(left), + ts_subtree_dynamic_precedence(left)); return true; } - if (left->dynamic_precedence > right->dynamic_precedence) { + if (ts_subtree_dynamic_precedence(left) > ts_subtree_dynamic_precedence(right)) { LOG("select_higher_precedence symbol:%s, prec:%u, over_symbol:%s, other_prec:%u", - SYM_NAME(left->symbol), left->dynamic_precedence, SYM_NAME(right->symbol), - right->dynamic_precedence); + TREE_NAME(left), ts_subtree_dynamic_precedence(left), TREE_NAME(right), + ts_subtree_dynamic_precedence(right)); return false; } - if (left->error_cost > 0) return true; + if (ts_subtree_error_cost(left) > 0) return true; int comparison = ts_subtree_compare(left, right); switch (comparison) { case -1: - LOG("select_earlier symbol:%s, over_symbol:%s", SYM_NAME(left->symbol), - SYM_NAME(right->symbol)); + LOG("select_earlier symbol:%s, over_symbol:%s", TREE_NAME(left), TREE_NAME(right)); return false; break; case 1: - LOG("select_earlier symbol:%s, over_symbol:%s", SYM_NAME(right->symbol), - SYM_NAME(left->symbol)); + LOG("select_earlier symbol:%s, over_symbol:%s", TREE_NAME(right), TREE_NAME(left)); return true; default: - LOG("select_existing symbol:%s, over_symbol:%s", SYM_NAME(left->symbol), - SYM_NAME(right->symbol)); + LOG("select_existing symbol:%s, over_symbol:%s", TREE_NAME(left), TREE_NAME(right)); return false; } } static void ts_parser__shift(TSParser *self, StackVersion version, TSStateId state, - const Subtree *lookahead, bool extra) { - const Subtree *subtree_to_push; - if (extra != lookahead->extra) { - Subtree *result = ts_subtree_make_mut(&self->tree_pool, lookahead); - result->extra = extra; - subtree_to_push = result; + Subtree lookahead, bool extra) { + Subtree subtree_to_push; + if (extra != ts_subtree_extra(lookahead)) { + MutableSubtree result = ts_subtree_make_mut(&self->tree_pool, lookahead); + ts_subtree_set_extra(&result); + subtree_to_push = ts_subtree_from_mut(result); } else { subtree_to_push = lookahead; } - bool is_pending = subtree_to_push->child_count > 0; + bool is_pending = ts_subtree_child_count(subtree_to_push) > 0; ts_stack_push(self->stack, version, subtree_to_push, is_pending, state); - if (subtree_to_push->has_external_tokens) { + if (ts_subtree_has_external_tokens(subtree_to_push)) { ts_stack_set_last_external_token( self->stack, version, ts_subtree_last_external_token(subtree_to_push) ); } } -static bool ts_parser__replace_children(TSParser *self, Subtree *tree, SubtreeArray *children) { - self->scratch_tree = *tree; - self->scratch_tree.child_count = 0; - ts_subtree_set_children(&self->scratch_tree, children->contents, children->size, self->language); - if (ts_parser__select_tree(self, tree, &self->scratch_tree)) { - *tree = self->scratch_tree; +static bool ts_parser__replace_children(TSParser *self, MutableSubtree *tree, SubtreeArray *children) { + *self->scratch_tree.ptr = *tree->ptr; + self->scratch_tree.ptr->child_count = 0; + ts_subtree_set_children(self->scratch_tree, children->contents, children->size, self->language); + if (ts_parser__select_tree(self, ts_subtree_from_mut(*tree), ts_subtree_from_mut(self->scratch_tree))) { + *tree->ptr = *self->scratch_tree.ptr; return true; } else { return false; @@ -675,11 +674,11 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version, TSSy // node. They will be re-pushed onto the stack after the parent node is // created and pushed. SubtreeArray children = slice.subtrees; - while (children.size > 0 && children.contents[children.size - 1]->extra) { + while (children.size > 0 && ts_subtree_extra(children.contents[children.size - 1])) { children.size--; } - Subtree *parent = ts_subtree_new_node(&self->tree_pool, + MutableSubtree parent = ts_subtree_new_node(&self->tree_pool, symbol, &children, alias_sequence_id, self->language ); @@ -693,11 +692,11 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version, TSSy i++; SubtreeArray children = next_slice.subtrees; - while (children.size > 0 && children.contents[children.size - 1]->extra) { + while (children.size > 0 && ts_subtree_extra(children.contents[children.size - 1])) { children.size--; } - if (ts_parser__replace_children(self, parent, &children)) { + if (ts_parser__replace_children(self, &parent, &children)) { ts_subtree_array_delete(&self->tree_pool, &slice.subtrees); slice = next_slice; } else { @@ -705,23 +704,23 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version, TSSy } } - parent->dynamic_precedence += dynamic_precedence; - parent->alias_sequence_id = alias_sequence_id; + parent.ptr->dynamic_precedence += dynamic_precedence; + parent.ptr->alias_sequence_id = alias_sequence_id; TSStateId state = ts_stack_state(self->stack, slice_version); TSStateId next_state = ts_language_next_state(self->language, state, symbol); if (fragile || pop.size > 1 || initial_version_count > 1) { - parent->fragile_left = true; - parent->fragile_right = true; - parent->parse_state = TS_TREE_STATE_NONE; + parent.ptr->fragile_left = true; + parent.ptr->fragile_right = true; + parent.ptr->parse_state = TS_TREE_STATE_NONE; } else { - parent->parse_state = state; + parent.ptr->parse_state = state; } // Push the parent node onto the stack, along with any extra tokens that // were previously on top of the stack. - ts_stack_push(self->stack, slice_version, parent, false, next_state); - for (uint32_t j = parent->child_count; j < slice.subtrees.size; j++) { + ts_stack_push(self->stack, slice_version, ts_subtree_from_mut(parent), false, next_state); + for (uint32_t j = parent.ptr->child_count; j < slice.subtrees.size; j++) { ts_stack_push(self->stack, slice_version, slice.subtrees.contents[j], false, next_state); } @@ -740,35 +739,40 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version, TSSy : STACK_VERSION_NONE; } -static void ts_parser__accept(TSParser *self, StackVersion version, const Subtree *lookahead) { - assert(lookahead->symbol == ts_builtin_sym_end); +static void ts_parser__accept(TSParser *self, StackVersion version, Subtree lookahead) { + assert(ts_subtree_is_eof(lookahead)); ts_stack_push(self->stack, version, lookahead, false, 1); StackSliceArray pop = ts_stack_pop_all(self->stack, version); for (uint32_t i = 0; i < pop.size; i++) { SubtreeArray trees = pop.contents[i].subtrees; - const Subtree *root = NULL; + Subtree root = NULL_SUBTREE; for (uint32_t j = trees.size - 1; j + 1 > 0; j--) { - const Subtree *child = trees.contents[j]; - if (!child->extra) { - for (uint32_t k = 0; k < child->child_count; k++) { - ts_subtree_retain(child->children[k]); + Subtree child = trees.contents[j]; + if (!ts_subtree_extra(child)) { + assert(!child.data.is_inline); + uint32_t child_count = ts_subtree_child_count(child); + for (uint32_t k = 0; k < child_count; k++) { + ts_subtree_retain(child.ptr->children[k]); } - array_splice(&trees, j, 1, child->child_count, child->children); - root = ts_subtree_new_node( - &self->tree_pool, child->symbol, &trees, - child->alias_sequence_id, self->language - ); + array_splice(&trees, j, 1, child_count, child.ptr->children); + root = ts_subtree_from_mut(ts_subtree_new_node( + &self->tree_pool, + ts_subtree_symbol(child), + &trees, + child.ptr->alias_sequence_id, + self->language + )); ts_subtree_release(&self->tree_pool, child); break; } } - assert(root && root->ref_count > 0); + assert(root.ptr); self->accept_count++; - if (self->finished_tree) { + if (self->finished_tree.ptr) { if (ts_parser__select_tree(self, self->finished_tree, root)) { ts_subtree_release(&self->tree_pool, self->finished_tree); self->finished_tree = root; @@ -906,7 +910,7 @@ static void ts_parser__handle_error(TSParser *self, StackVersion version, Length padding = length_sub(self->lexer.token_end_position, position); StackVersion version_with_missing_tree = ts_stack_copy_version(self->stack, v); - const Subtree *missing_tree = ts_subtree_new_missing_leaf( + Subtree missing_tree = ts_subtree_new_missing_leaf( &self->tree_pool, missing_symbol, padding, self->language ); ts_stack_push( @@ -931,7 +935,7 @@ static void ts_parser__handle_error(TSParser *self, StackVersion version, } } - ts_stack_push(self->stack, v, NULL, false, ERROR_STATE); + ts_stack_push(self->stack, v, NULL_SUBTREE, false, ERROR_STATE); v = (v == version) ? previous_version_count : v + 1; } @@ -953,15 +957,23 @@ static void ts_parser__halt_parse(TSParser *self) { ts_stack_position(self->stack, 0) ); - Subtree *filler_node = ts_subtree_new_error(&self->tree_pool, remaining_length, length_zero(), 0, self->language); - filler_node->visible = false; + Subtree filler_node = ts_subtree_new_error( + &self->tree_pool, + 0, + length_zero(), + remaining_length, + remaining_length.bytes, + 0, + self->language + ); + ts_subtree_to_mut_unsafe(filler_node).ptr->visible = false; ts_stack_push(self->stack, 0, filler_node, false, 0); SubtreeArray children = array_new(); - Subtree *root_error = ts_subtree_new_error_node(&self->tree_pool, &children, self->language); + Subtree root_error = ts_subtree_new_error_node(&self->tree_pool, &children, false, self->language); ts_stack_push(self->stack, 0, root_error, false, 0); - Subtree *eof = ts_subtree_new_leaf( + Subtree eof = ts_subtree_new_leaf( &self->tree_pool, ts_builtin_sym_end, length_zero(), @@ -999,10 +1011,13 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, un SubtreeArray error_trees = ts_stack_pop_error(self->stack, slice.version); if (error_trees.size > 0) { assert(error_trees.size == 1); - const Subtree *error_tree = error_trees.contents[0]; - array_splice(&slice.subtrees, 0, 0, error_tree->child_count, error_tree->children); - for (unsigned j = 0; j < error_tree->child_count; j++) { - ts_subtree_retain(slice.subtrees.contents[j]); + Subtree error_tree = error_trees.contents[0]; + uint32_t error_child_count = ts_subtree_child_count(error_tree); + if (error_child_count > 0) { + array_splice(&slice.subtrees, 0, 0, error_child_count, error_tree.ptr->children); + for (unsigned j = 0; j < error_child_count; j++) { + ts_subtree_retain(slice.subtrees.contents[j]); + } } ts_subtree_array_delete(&self->tree_pool, &error_trees); } @@ -1010,15 +1025,14 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, un SubtreeArray trailing_extras = ts_subtree_array_remove_trailing_extras(&slice.subtrees); if (slice.subtrees.size > 0) { - Subtree *error = ts_subtree_new_error_node(&self->tree_pool, &slice.subtrees, self->language); - error->extra = true; + Subtree error = ts_subtree_new_error_node(&self->tree_pool, &slice.subtrees, true, self->language); ts_stack_push(self->stack, slice.version, error, false, goal_state); } else { array_delete(&slice.subtrees); } for (unsigned j = 0; j < trailing_extras.size; j++) { - const Subtree *tree = trailing_extras.contents[j]; + Subtree tree = trailing_extras.contents[j]; ts_stack_push(self->stack, slice.version, tree, false, goal_state); } @@ -1029,7 +1043,7 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, un return previous_version != STACK_VERSION_NONE; } -static void ts_parser__recover(TSParser *self, StackVersion version, const Subtree *lookahead) { +static void ts_parser__recover(TSParser *self, StackVersion version, Subtree lookahead) { bool did_recover = false; unsigned previous_version_count = ts_stack_version_count(self->stack); Length position = ts_stack_position(self->stack, version); @@ -1037,7 +1051,7 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr unsigned node_count_since_error = ts_stack_node_count_since_error(self->stack, version); unsigned current_error_cost = ts_stack_error_cost(self->stack, version); - if (summary && lookahead->symbol != ts_builtin_sym_error) { + if (summary && !ts_subtree_is_error(lookahead)) { for (unsigned i = 0; i < summary->size; i++) { StackSummaryEntry entry = summary->contents[i]; @@ -1066,7 +1080,7 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr (position.extent.row - entry.position.extent.row) * ERROR_COST_PER_SKIPPED_LINE; if (ts_parser__better_version_exists(self, version, false, new_cost)) break; - if (ts_language_has_actions(self->language, entry.state, lookahead->symbol)) { + if (ts_language_has_actions(self->language, entry.state, ts_subtree_symbol(lookahead))) { if (ts_parser__recover_to_state(self, version, depth, entry.state)) { did_recover = true; LOG("recover_to_previous state:%u, depth:%u", entry.state, depth); @@ -1089,10 +1103,10 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr return; } - if (lookahead->symbol == ts_builtin_sym_end) { + if (ts_subtree_is_eof(lookahead)) { LOG("recover_eof"); SubtreeArray children = array_new(); - const Subtree *parent = ts_subtree_new_error_node(&self->tree_pool, &children, self->language); + Subtree parent = ts_subtree_new_error_node(&self->tree_pool, &children, false, self->language); ts_stack_push(self->stack, version, parent, false, 1); ts_parser__accept(self, version, lookahead); return; @@ -1110,18 +1124,18 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr } unsigned n; - const TSParseAction *actions = ts_language_actions(self->language, 1, lookahead->symbol, &n); + const TSParseAction *actions = ts_language_actions(self->language, 1, ts_subtree_symbol(lookahead), &n); if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.extra) { - Subtree *mutable_lookahead = ts_subtree_make_mut(&self->tree_pool, lookahead); - mutable_lookahead->extra = true; - lookahead = mutable_lookahead; + MutableSubtree mutable_lookahead = ts_subtree_make_mut(&self->tree_pool, lookahead); + ts_subtree_set_extra(&mutable_lookahead); + lookahead = ts_subtree_from_mut(mutable_lookahead); } - LOG("skip_token symbol:%s", SYM_NAME(lookahead->symbol)); + LOG("skip_token symbol:%s", TREE_NAME(lookahead)); SubtreeArray children = array_new(); array_reserve(&children, 1); array_push(&children, lookahead); - const Subtree *error_repeat = ts_subtree_new_node( + MutableSubtree error_repeat = ts_subtree_new_node( &self->tree_pool, ts_builtin_sym_error_repeat, &children, @@ -1134,7 +1148,7 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr assert(pop.size == 1); assert(pop.contents[0].subtrees.size == 1); ts_stack_renumber_version(self->stack, pop.contents[0].version, version); - array_push(&pop.contents[0].subtrees, error_repeat); + array_push(&pop.contents[0].subtrees, ts_subtree_from_mut(error_repeat)); error_repeat = ts_subtree_new_node( &self->tree_pool, ts_builtin_sym_error_repeat, @@ -1144,9 +1158,9 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr ); } - ts_stack_push(self->stack, version, error_repeat, false, ERROR_STATE); + ts_stack_push(self->stack, version, ts_subtree_from_mut(error_repeat), false, ERROR_STATE); - if (lookahead->has_external_tokens) { + if (ts_subtree_has_external_tokens(lookahead)) { ts_stack_set_last_external_token( self->stack, version, ts_subtree_last_external_token(lookahead) ); @@ -1156,11 +1170,11 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_node_reuse) { TSStateId state = ts_stack_state(self->stack, version); uint32_t position = ts_stack_position(self->stack, version).bytes; - const Subtree *last_external_token = ts_stack_last_external_token(self->stack, version); + Subtree last_external_token = ts_stack_last_external_token(self->stack, version); bool did_reuse = true; - const Subtree *lookahead = NULL; - TableEntry table_entry; + Subtree lookahead = NULL_SUBTREE; + TableEntry table_entry = {.action_count = 0}; // If possible, reuse a node from the previous syntax tree. if (allow_node_reuse) { @@ -1170,7 +1184,7 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_ } // Otherwise, try to reuse the token previously returned by the lexer. - if (!lookahead) { + if (!lookahead.ptr) { did_reuse = false; lookahead = ts_parser__get_cached_token( self, state, position, last_external_token, &table_entry @@ -1178,10 +1192,10 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_ } // Otherwise, re-run the lexer. - if (!lookahead) { + if (!lookahead.ptr) { lookahead = ts_parser__lex(self, version, state); ts_parser__set_cached_token(self, position, last_external_token, lookahead); - ts_language_table_entry(self->language, state, lookahead->symbol, &table_entry); + ts_language_table_entry(self->language, state, ts_subtree_symbol(lookahead), &table_entry); } for (;;) { @@ -1206,9 +1220,9 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_ LOG("shift state:%u", next_state); } - if (lookahead->child_count > 0) { + if (ts_subtree_child_count(lookahead) > 0) { ts_parser__breakdown_lookahead(self, &lookahead, state, &self->reusable_node); - next_state = ts_language_next_state(self->language, state, lookahead->symbol); + next_state = ts_language_next_state(self->language, state, ts_subtree_symbol(lookahead)); } ts_parser__shift(self, version, next_state, lookahead, action.params.extra); @@ -1237,7 +1251,7 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_ } case TSParseActionTypeRecover: { - if (lookahead->child_count > 0) { + if (ts_subtree_child_count(lookahead) > 0) { ts_parser__breakdown_lookahead(self, &lookahead, ERROR_STATE, &self->reusable_node); } @@ -1261,18 +1275,21 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_ continue; } - if (lookahead->is_keyword && lookahead->symbol != self->language->keyword_capture_token) { + if ( + ts_subtree_is_keyword(lookahead) && + ts_subtree_symbol(lookahead) != self->language->keyword_capture_token + ) { ts_language_table_entry(self->language, state, self->language->keyword_capture_token, &table_entry); if (table_entry.action_count > 0) { LOG( "switch from_keyword:%s, to_word_token:%s", - SYM_NAME(lookahead->symbol), + TREE_NAME(lookahead), SYM_NAME(self->language->keyword_capture_token) ); - Subtree *mutable_lookahead = ts_subtree_make_mut(&self->tree_pool, lookahead); - mutable_lookahead->symbol = self->language->keyword_capture_token; - lookahead = mutable_lookahead; + MutableSubtree mutable_lookahead = ts_subtree_make_mut(&self->tree_pool, lookahead); + ts_subtree_set_symbol(&mutable_lookahead, self->language->keyword_capture_token); + lookahead = ts_subtree_from_mut(mutable_lookahead); continue; } } @@ -1395,14 +1412,15 @@ TSParser *ts_parser_new() { array_reserve(&self->reduce_actions, 4); self->tree_pool = ts_subtree_pool_new(32); self->stack = ts_stack_new(&self->tree_pool); - self->finished_tree = NULL; + self->finished_tree = NULL_SUBTREE; self->reusable_node = reusable_node_new(); self->dot_graph_file = NULL; self->halt_on_error = false; self->enabled = true; self->operation_limit = SIZE_MAX; - self->old_tree = NULL; - ts_parser__set_cached_token(self, 0, NULL, NULL); + self->old_tree = NULL_SUBTREE; + self->scratch_tree.ptr = &self->scratch_tree_data; + ts_parser__set_cached_token(self, 0, NULL_SUBTREE, NULL_SUBTREE); return self; } @@ -1411,12 +1429,12 @@ void ts_parser_delete(TSParser *self) { if (self->reduce_actions.contents) { array_delete(&self->reduce_actions); } - if (self->old_tree) { + if (self->old_tree.ptr) { ts_subtree_release(&self->tree_pool, self->old_tree); - self->old_tree = NULL; + self->old_tree = NULL_SUBTREE; } ts_lexer_delete(&self->lexer); - ts_parser__set_cached_token(self, 0, NULL, NULL); + ts_parser__set_cached_token(self, 0, NULL_SUBTREE, NULL_SUBTREE); ts_subtree_pool_delete(&self->tree_pool); reusable_node_delete(&self->reusable_node); ts_parser_set_language(self, NULL); @@ -1489,18 +1507,18 @@ void ts_parser_reset(TSParser *self) { self->language->external_scanner.deserialize(self->external_scanner_payload, NULL, 0); } - if (self->old_tree) { + if (self->old_tree.ptr) { ts_subtree_release(&self->tree_pool, self->old_tree); - self->old_tree = NULL; + self->old_tree = NULL_SUBTREE; } reusable_node_clear(&self->reusable_node); ts_lexer_reset(&self->lexer, length_zero()); ts_stack_clear(self->stack); - ts_parser__set_cached_token(self, 0, NULL, NULL); - if (self->finished_tree) { + ts_parser__set_cached_token(self, 0, NULL_SUBTREE, NULL_SUBTREE); + if (self->finished_tree.ptr) { ts_subtree_release(&self->tree_pool, self->finished_tree); - self->finished_tree = NULL; + self->finished_tree = NULL_SUBTREE; } self->accept_count = 0; } @@ -1552,7 +1570,7 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) { } unsigned min_error_cost = ts_parser__condense_stack(self); - if (self->finished_tree && self->finished_tree->error_cost < min_error_cost) { + if (self->finished_tree.ptr && ts_subtree_error_cost(self->finished_tree) < min_error_cost) { break; } else if (self->halt_on_error && min_error_cost > 0) { ts_parser__halt_parse(self); @@ -1565,7 +1583,7 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) { LOG_TREE(); TSTree *result = ts_tree_new(self->finished_tree, self->language); - self->finished_tree = NULL; + self->finished_tree = NULL_SUBTREE; ts_parser_reset(self); return result; } diff --git a/src/runtime/reusable_node.h b/src/runtime/reusable_node.h index 3a915fcc..8c18bce7 100644 --- a/src/runtime/reusable_node.h +++ b/src/runtime/reusable_node.h @@ -1,26 +1,26 @@ #include "runtime/subtree.h" typedef struct { - const Subtree *tree; + Subtree tree; uint32_t child_index; uint32_t byte_offset; } StackEntry; typedef struct { Array(StackEntry) stack; - const Subtree *last_external_token; + Subtree last_external_token; } ReusableNode; static inline ReusableNode reusable_node_new() { - return (ReusableNode) {array_new(), NULL}; + return (ReusableNode) {array_new(), (Subtree) {.ptr = NULL}}; } static inline void reusable_node_clear(ReusableNode *self) { array_clear(&self->stack); - self->last_external_token = NULL; + self->last_external_token.ptr = NULL; } -static inline void reusable_node_reset(ReusableNode *self, const Subtree *tree) { +static inline void reusable_node_reset(ReusableNode *self, Subtree tree) { reusable_node_clear(self); array_push(&self->stack, ((StackEntry) { .tree = tree, @@ -29,10 +29,10 @@ static inline void reusable_node_reset(ReusableNode *self, const Subtree *tree) })); } -static inline const Subtree *reusable_node_tree(ReusableNode *self) { +static inline Subtree reusable_node_tree(ReusableNode *self) { return self->stack.size > 0 ? self->stack.contents[self->stack.size - 1].tree - : NULL; + : (Subtree) { .ptr = NULL }; } static inline uint32_t reusable_node_byte_offset(ReusableNode *self) { @@ -48,21 +48,21 @@ static inline void reusable_node_delete(ReusableNode *self) { static inline void reusable_node_advance(ReusableNode *self) { StackEntry last_entry = *array_back(&self->stack); uint32_t byte_offset = last_entry.byte_offset + ts_subtree_total_bytes(last_entry.tree); - if (last_entry.tree->has_external_tokens) { + if (ts_subtree_has_external_tokens(last_entry.tree)) { self->last_external_token = ts_subtree_last_external_token(last_entry.tree); } - const Subtree *tree; + Subtree tree; uint32_t next_index; do { StackEntry popped_entry = array_pop(&self->stack); next_index = popped_entry.child_index + 1; if (self->stack.size == 0) return; tree = array_back(&self->stack)->tree; - } while (tree->child_count <= next_index); + } while (ts_subtree_child_count(tree) <= next_index); array_push(&self->stack, ((StackEntry) { - .tree = tree->children[next_index], + .tree = tree.ptr->children[next_index], .child_index = next_index, .byte_offset = byte_offset, })); @@ -70,9 +70,9 @@ static inline void reusable_node_advance(ReusableNode *self) { static inline bool reusable_node_descend(ReusableNode *self) { StackEntry last_entry = *array_back(&self->stack); - if (last_entry.tree->child_count > 0) { + if (ts_subtree_child_count(last_entry.tree) > 0) { array_push(&self->stack, ((StackEntry) { - .tree = last_entry.tree->children[0], + .tree = last_entry.tree.ptr->children[0], .child_index = 0, .byte_offset = last_entry.byte_offset, })); diff --git a/src/runtime/stack.c b/src/runtime/stack.c index 39321964..7645bd4a 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -21,7 +21,7 @@ typedef struct StackNode StackNode; typedef struct { StackNode *node; - const Subtree *subtree; + Subtree subtree; bool is_pending; } StackLink; @@ -58,7 +58,7 @@ typedef enum { typedef struct { StackNode *node; - const Subtree *last_external_token; + Subtree last_external_token; StackSummary *summary; unsigned node_count_at_last_error; TSSymbol lookahead_when_paused; @@ -100,10 +100,12 @@ recur: StackNode *first_predecessor = NULL; if (self->link_count > 0) { for (unsigned i = self->link_count - 1; i > 0; i--) { - if (self->links[i].subtree) ts_subtree_release(subtree_pool, self->links[i].subtree); - stack_node_release(self->links[i].node, pool, subtree_pool); + StackLink link = self->links[i]; + if (link.subtree.ptr) ts_subtree_release(subtree_pool, link.subtree); + stack_node_release(link.node, pool, subtree_pool); } - if (self->links[0].subtree) ts_subtree_release(subtree_pool, self->links[0].subtree); + StackLink link = self->links[0]; + if (link.subtree.ptr) ts_subtree_release(subtree_pool, link.subtree); first_predecessor = self->links[0].node; } @@ -119,7 +121,7 @@ recur: } } -static StackNode *stack_node_new(StackNode *previous_node, const Subtree *subtree, +static StackNode *stack_node_new(StackNode *previous_node, Subtree subtree, bool is_pending, TSStateId state, StackNodeArray *pool) { StackNode *node = pool->size > 0 ? array_pop(pool) : @@ -139,15 +141,11 @@ static StackNode *stack_node_new(StackNode *previous_node, const Subtree *subtre node->dynamic_precedence = previous_node->dynamic_precedence; node->node_count = previous_node->node_count; - if (subtree) { - node->error_cost += subtree->error_cost; + if (subtree.ptr) { + node->error_cost += ts_subtree_error_cost(subtree); node->position = length_add(node->position, ts_subtree_total_size(subtree)); - if (subtree->child_count) { - node->node_count += subtree->node_count; - node->dynamic_precedence += subtree->dynamic_precedence; - } else { - node->node_count++; - } + node->node_count += ts_subtree_node_count(subtree); + node->dynamic_precedence += ts_subtree_dynamic_precedence(subtree); } } else { node->position = length_zero(); @@ -157,17 +155,16 @@ static StackNode *stack_node_new(StackNode *previous_node, const Subtree *subtre return node; } -static bool stack__subtree_is_equivalent(const Subtree *left, const Subtree *right) { +static bool stack__subtree_is_equivalent(Subtree left, Subtree right) { return - left == right || - (left && - right && - left->symbol == right->symbol && - ((left->error_cost > 0 && right->error_cost > 0) || - (left->padding.bytes == right->padding.bytes && - left->size.bytes == right->size.bytes && - left->child_count == right->child_count && - left->extra == right->extra && + left.ptr == right.ptr || + (left.ptr && right.ptr && + ts_subtree_symbol(left) == ts_subtree_symbol(right) && + ((ts_subtree_error_cost(left) > 0 && ts_subtree_error_cost(right) > 0) || + (ts_subtree_padding(left).bytes == ts_subtree_padding(right).bytes && + ts_subtree_size(left).bytes == ts_subtree_size(right).bytes && + ts_subtree_child_count(left) == ts_subtree_child_count(right) && + ts_subtree_extra(left) == ts_subtree_extra(right) && ts_subtree_external_scanner_state_eq(left, right)))); } @@ -183,13 +180,14 @@ static void stack_node_add_link(StackNode *self, StackLink link, SubtreePool *su // we can safely remove the ambiguity ahead of time without changing behavior. if (existing_link->node == link.node) { if ( - link.subtree->child_count > 0 && - link.subtree->dynamic_precedence > existing_link->subtree->dynamic_precedence + ts_subtree_dynamic_precedence(link.subtree) > + ts_subtree_dynamic_precedence(existing_link->subtree) ) { ts_subtree_retain(link.subtree); ts_subtree_release(subtree_pool, existing_link->subtree); existing_link->subtree = link.subtree; - self->dynamic_precedence = link.node->dynamic_precedence + link.subtree->dynamic_precedence; + self->dynamic_precedence = + link.node->dynamic_precedence + ts_subtree_dynamic_precedence(link.subtree); } return; } @@ -200,8 +198,10 @@ static void stack_node_add_link(StackNode *self, StackLink link, SubtreePool *su for (int j = 0; j < link.node->link_count; j++) { stack_node_add_link(existing_link->node, link.node->links[j], subtree_pool); } - int dynamic_precedence = link.node->dynamic_precedence; - if (link.subtree) dynamic_precedence += link.subtree->dynamic_precedence; + int32_t dynamic_precedence = link.node->dynamic_precedence; + if (link.subtree.ptr) { + dynamic_precedence += ts_subtree_dynamic_precedence(link.subtree); + } if (dynamic_precedence > self->dynamic_precedence) { self->dynamic_precedence = dynamic_precedence; } @@ -217,14 +217,10 @@ static void stack_node_add_link(StackNode *self, StackLink link, SubtreePool *su int dynamic_precedence = link.node->dynamic_precedence; self->links[self->link_count++] = link; - if (link.subtree) { + if (link.subtree.ptr) { ts_subtree_retain(link.subtree); - if (link.subtree->child_count > 0) { - node_count += link.subtree->node_count; - dynamic_precedence += link.subtree->dynamic_precedence; - } else { - node_count++; - } + node_count += ts_subtree_node_count(link.subtree); + dynamic_precedence += ts_subtree_dynamic_precedence(link.subtree); } if (node_count > self->node_count) self->node_count = node_count; @@ -233,7 +229,7 @@ static void stack_node_add_link(StackNode *self, StackLink link, SubtreePool *su static void stack_head_delete(StackHead *self, StackNodeArray *pool, SubtreePool *subtree_pool) { if (self->node) { - if (self->last_external_token) { + if (self->last_external_token.ptr) { ts_subtree_release(subtree_pool, self->last_external_token); } if (self->summary) { @@ -255,7 +251,7 @@ static StackVersion ts_stack__add_version(Stack *self, StackVersion original_ver }; array_push(&self->heads, head); stack_node_retain(node); - if (head.last_external_token) ts_subtree_retain(head.last_external_token); + if (head.last_external_token.ptr) ts_subtree_retain(head.last_external_token); return (StackVersion)(self->heads.size - 1); } @@ -343,13 +339,13 @@ inline StackSliceArray stack__iter(Stack *self, StackVersion version, } next_iterator->node = link.node; - if (link.subtree) { + if (link.subtree.ptr) { if (include_subtrees) { array_push(&next_iterator->subtrees, link.subtree); ts_subtree_retain(link.subtree); } - if (!link.subtree->extra) { + if (!ts_subtree_extra(link.subtree)) { next_iterator->subtree_count++; if (!link.is_pending) { next_iterator->is_pending = false; @@ -379,7 +375,7 @@ Stack *ts_stack_new(SubtreePool *subtree_pool) { array_reserve(&self->node_pool, MAX_NODE_POOL_SIZE); self->subtree_pool = subtree_pool; - self->base_node = stack_node_new(NULL, NULL, false, 1, &self->node_pool); + self->base_node = stack_node_new(NULL, (Subtree) {.ptr = NULL}, false, 1, &self->node_pool); ts_stack_clear(self); return self; @@ -416,14 +412,14 @@ Length ts_stack_position(const Stack *self, StackVersion version) { return array_get(&self->heads, version)->node->position; } -const Subtree *ts_stack_last_external_token(const Stack *self, StackVersion version) { +Subtree ts_stack_last_external_token(const Stack *self, StackVersion version) { return array_get(&self->heads, version)->last_external_token; } -void ts_stack_set_last_external_token(Stack *self, StackVersion version, const Subtree *token) { +void ts_stack_set_last_external_token(Stack *self, StackVersion version, Subtree token) { StackHead *head = array_get(&self->heads, version); - if (token) ts_subtree_retain(token); - if (head->last_external_token) ts_subtree_release(self->subtree_pool, head->last_external_token); + if (token.ptr) ts_subtree_retain(token); + if (head->last_external_token.ptr) ts_subtree_release(self->subtree_pool, head->last_external_token); head->last_external_token = token; } @@ -432,7 +428,7 @@ unsigned ts_stack_error_cost(const Stack *self, StackVersion version) { unsigned result = head->node->error_cost; if ( head->status == StackStatusPaused || - (head->node->state == ERROR_STATE && !head->node->links[0].subtree)) { + (head->node->state == ERROR_STATE && !head->node->links[0].subtree.ptr)) { result += ERROR_COST_PER_RECOVERY; } return result; @@ -446,11 +442,11 @@ unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version return head->node->node_count - head->node_count_at_last_error; } -void ts_stack_push(Stack *self, StackVersion version, const Subtree *subtree, +void ts_stack_push(Stack *self, StackVersion version, Subtree subtree, bool pending, TSStateId state) { StackHead *head = array_get(&self->heads, version); StackNode *new_node = stack_node_new(head->node, subtree, pending, state, &self->node_pool); - if (!subtree) head->node_count_at_last_error = new_node->node_count; + if (!subtree.ptr) head->node_count_at_last_error = new_node->node_count; head->node = new_node; } @@ -507,7 +503,7 @@ StackSliceArray ts_stack_pop_pending(Stack *self, StackVersion version) { inline StackAction pop_error_callback(void *payload, const Iterator *iterator) { if (iterator->subtrees.size > 0) { bool *found_error = payload; - if (!*found_error && iterator->subtrees.contents[0]->symbol == ts_builtin_sym_error) { + if (!*found_error && ts_subtree_is_error(iterator->subtrees.contents[0])) { *found_error = true; return StackActionPop | StackActionStop; } else { @@ -521,7 +517,7 @@ inline StackAction pop_error_callback(void *payload, const Iterator *iterator) { SubtreeArray ts_stack_pop_error(Stack *self, StackVersion version) { StackNode *node = array_get(&self->heads, version)->node; for (unsigned i = 0; i < node->link_count; i++) { - if (node->links[i].subtree && node->links[i].subtree->symbol == ts_builtin_sym_error) { + if (node->links[i].subtree.ptr && ts_subtree_is_error(node->links[i].subtree)) { bool found_error = false; StackSliceArray pop = stack__iter(self, version, pop_error_callback, &found_error, 1); if (pop.size > 0) { @@ -615,7 +611,7 @@ StackVersion ts_stack_copy_version(Stack *self, StackVersion version) { array_push(&self->heads, self->heads.contents[version]); StackHead *head = array_back(&self->heads); stack_node_retain(head->node); - if (head->last_external_token) ts_subtree_retain(head->last_external_token); + if (head->last_external_token.ptr) ts_subtree_retain(head->last_external_token); head->summary = NULL; return self->heads.size - 1; } @@ -686,7 +682,7 @@ void ts_stack_clear(Stack *self) { array_clear(&self->heads); array_push(&self->heads, ((StackHead){ .node = self->base_node, - .last_external_token = NULL, + .last_external_token.ptr = NULL, .status = StackStatusActive, .lookahead_when_paused = 0, })); @@ -721,8 +717,8 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) ts_stack_error_cost(self, i) ); - if (head->last_external_token) { - const ExternalScannerState *state = &head->last_external_token->external_scanner_state; + if (head->last_external_token.ptr) { + const ExternalScannerState *state = &head->last_external_token.ptr->external_scanner_state; const char *data = ts_external_scanner_state_data(state); fprintf(f, "\nexternal_scanner_state:"); for (uint32_t j = 0; j < state->length; j++) fprintf(f, " %2X", data[j]); @@ -751,13 +747,17 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) all_iterators_done = false; fprintf(f, "node_%p [", node); - if (node->state == ERROR_STATE) + if (node->state == ERROR_STATE) { fprintf(f, "label=\"?\""); - else if (node->link_count == 1 && node->links[0].subtree && - node->links[0].subtree->extra) + } else if ( + node->link_count == 1 && + node->links[0].subtree.ptr && + ts_subtree_extra(node->links[0].subtree) + ) { fprintf(f, "shape=point margin=0 label=\"\""); - else + } else { fprintf(f, "label=\"%d\"", node->state); + } fprintf( f, @@ -772,26 +772,28 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) for (int j = 0; j < node->link_count; j++) { StackLink link = node->links[j]; fprintf(f, "node_%p -> node_%p [", node, link.node); - if (link.is_pending) - fprintf(f, "style=dashed "); - if (link.subtree && link.subtree->extra) - fprintf(f, "fontcolor=gray "); + if (link.is_pending) fprintf(f, "style=dashed "); + if (link.subtree.ptr && ts_subtree_extra(link.subtree)) fprintf(f, "fontcolor=gray "); - if (!link.subtree) { + if (!link.subtree.ptr) { fprintf(f, "color=red"); } else { fprintf(f, "label=\""); - if (link.subtree->visible && !link.subtree->named) fprintf(f, "'"); - const char *name = ts_language_symbol_name(language, link.subtree->symbol); + bool quoted = ts_subtree_visible(link.subtree) && !ts_subtree_named(link.subtree); + if (quoted) fprintf(f, "'"); + const char *name = ts_language_symbol_name(language, ts_subtree_symbol(link.subtree)); for (const char *c = name; *c; c++) { if (*c == '\"' || *c == '\\') fprintf(f, "\\"); fprintf(f, "%c", *c); } - if (link.subtree->visible && !link.subtree->named) fprintf(f, "'"); + if (quoted) fprintf(f, "'"); fprintf(f, "\""); - fprintf(f, "labeltooltip=\"error_cost: %u\ndynamic_precedence: %u\"", - link.subtree->error_cost, - link.subtree->dynamic_precedence); + fprintf( + f, + "labeltooltip=\"error_cost: %u\ndynamic_precedence: %u\"", + ts_subtree_error_cost(link.subtree), + ts_subtree_dynamic_precedence(link.subtree) + ); } fprintf(f, "];\n"); diff --git a/src/runtime/stack.h b/src/runtime/stack.h index b7dba342..dab0360d 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -42,10 +42,10 @@ uint32_t ts_stack_version_count(const Stack *); TSStateId ts_stack_state(const Stack *, StackVersion); // Get the last external token associated with a given version of the stack. -const Subtree *ts_stack_last_external_token(const Stack *, StackVersion); +Subtree ts_stack_last_external_token(const Stack *, StackVersion); // Set the last external token associated with a given version of the stack. -void ts_stack_set_last_external_token(Stack *, StackVersion, const Subtree *); +void ts_stack_set_last_external_token(Stack *, StackVersion, Subtree ); // Get the position of the given version of the stack within the document. Length ts_stack_position(const Stack *, StackVersion); @@ -55,7 +55,7 @@ Length ts_stack_position(const Stack *, StackVersion); // This transfers ownership of the tree to the Stack. Callers that // need to retain ownership of the tree for their own purposes should // first retain the tree. -void ts_stack_push(Stack *, StackVersion, const Subtree *, bool, TSStateId); +void ts_stack_push(Stack *, StackVersion, Subtree , bool, TSStateId); // Pop the given number of entries from the given version of the stack. This // operation can increase the number of stack versions by revealing multiple diff --git a/src/runtime/subtree.c b/src/runtime/subtree.c index 9b8967fc..f0eb2361 100644 --- a/src/runtime/subtree.c +++ b/src/runtime/subtree.c @@ -20,9 +20,17 @@ typedef struct { TSStateId TS_TREE_STATE_NONE = USHRT_MAX; -static const uint32_t MAX_TREE_POOL_SIZE = 0; -static const uint32_t SMALL_TREE_SIZE = offsetof(Subtree, children); -static const uint32_t LARGE_TREE_SIZE = sizeof(Subtree); +#ifdef TREE_SITTER_TEST + +#define TS_MAX_INLINE_TREE_LENGTH 2 +#define TS_MAX_TREE_POOL_SIZE 0 + +#else + +#define TS_MAX_INLINE_TREE_LENGTH UINT8_MAX +#define TS_MAX_TREE_POOL_SIZE 32 + +#endif static const ExternalScannerState empty_state = {.length = 0, .short_data = {0}}; @@ -75,8 +83,8 @@ void ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest) { dest->capacity = self.capacity; dest->contents = self.contents; if (self.capacity > 0) { - dest->contents = ts_calloc(self.capacity, sizeof(Subtree *)); - memcpy(dest->contents, self.contents, self.size * sizeof(Subtree *)); + dest->contents = ts_calloc(self.capacity, sizeof(Subtree)); + memcpy(dest->contents, self.contents, self.size * sizeof(Subtree)); for (uint32_t i = 0; i < self.size; i++) { ts_subtree_retain(dest->contents[i]); } @@ -95,8 +103,8 @@ SubtreeArray ts_subtree_array_remove_trailing_extras(SubtreeArray *self) { uint32_t i = self->size - 1; for (; i + 1 > 0; i--) { - const Subtree *child = self->contents[i]; - if (!child->extra) break; + Subtree child = self->contents[i]; + if (!ts_subtree_extra(child)) break; array_push(&result, child); } @@ -108,7 +116,7 @@ SubtreeArray ts_subtree_array_remove_trailing_extras(SubtreeArray *self) { void ts_subtree_array_reverse(SubtreeArray *self) { for (uint32_t i = 0, limit = self->size / 2; i < limit; i++) { size_t reverse_index = self->size - 1 - i; - const Subtree *swap = self->contents[i]; + Subtree swap = self->contents[i]; self->contents[i] = self->contents[reverse_index]; self->contents[reverse_index] = swap; } @@ -117,44 +125,32 @@ void ts_subtree_array_reverse(SubtreeArray *self) { // SubtreePool SubtreePool ts_subtree_pool_new(uint32_t capacity) { - SubtreePool self = {array_new(), array_new(), array_new()}; + SubtreePool self = {array_new(), array_new()}; array_reserve(&self.free_trees, capacity); - array_reserve(&self.free_small_trees, capacity); return self; } void ts_subtree_pool_delete(SubtreePool *self) { if (self->free_trees.contents) { for (unsigned i = 0; i < self->free_trees.size; i++) { - ts_free(self->free_trees.contents[i]); + ts_free(self->free_trees.contents[i].ptr); } array_delete(&self->free_trees); } - if (self->free_small_trees.contents) { - for (unsigned i = 0; i < self->free_small_trees.size; i++) { - ts_free(self->free_small_trees.contents[i]); - } - array_delete(&self->free_small_trees); - } if (self->tree_stack.contents) array_delete(&self->tree_stack); } -Subtree *ts_subtree_pool_allocate(SubtreePool *self, bool is_small) { - MutableSubtreeArray *array = is_small ? &self->free_small_trees : &self->free_trees; - Subtree *result; - if (array->size > 0) { - result = array_pop(array); +static SubtreeHeapData *ts_subtree_pool_allocate(SubtreePool *self) { + if (self->free_trees.size > 0) { + return array_pop(&self->free_trees).ptr; } else { - result = ts_malloc(is_small ? SMALL_TREE_SIZE : LARGE_TREE_SIZE); + return ts_malloc(sizeof(SubtreeHeapData)); } - result->is_small = is_small; - return result; } -void ts_subtree_pool_free(SubtreePool *self, Subtree *tree) { - MutableSubtreeArray *array = tree->is_small ? &self->free_small_trees : &self->free_trees; - if (array->capacity > 0 && array->size < MAX_TREE_POOL_SIZE) { - array_push(array, tree); +static void ts_subtree_pool_free(SubtreePool *self, SubtreeHeapData *tree) { + if (self->free_trees.capacity > 0 && self->free_trees.size + 1 <= TS_MAX_TREE_POOL_SIZE) { + array_push(&self->free_trees, (MutableSubtree) {.ptr = tree}); } else { ts_free(tree); } @@ -162,363 +158,401 @@ void ts_subtree_pool_free(SubtreePool *self, Subtree *tree) { // Subtree -Subtree *ts_subtree_new_leaf( - SubtreePool *pool, TSSymbol symbol, Length padding, Length size, uint32_t bytes_scanned, - TSStateId parse_state, bool has_external_tokens, bool is_keyword, const TSLanguage *language +static inline bool ts_subtree_can_inline(Length padding, Length size) { + return + padding.extent.row < 16 && + padding.extent.column < TS_MAX_INLINE_TREE_LENGTH && + size.extent.row == 0 && + size.extent.column < TS_MAX_INLINE_TREE_LENGTH; +} + +Subtree ts_subtree_new_leaf( + SubtreePool *pool, TSSymbol symbol, Length padding, Length size, + uint32_t bytes_scanned, TSStateId parse_state, bool has_external_tokens, + bool is_keyword, const TSLanguage *language ) { TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); - bool is_small = !has_external_tokens; - Subtree *result = ts_subtree_pool_allocate(pool, is_small); - result->ref_count = 1; - result->padding = padding; - result->size = size; - result->bytes_scanned = bytes_scanned; - result->error_cost = 0; - result->child_count = 0; - result->symbol = symbol; - result->parse_state = parse_state; - result->is_small = is_small; - result->visible = metadata.visible; - result->named = metadata.named; - result->extra = symbol == ts_builtin_sym_end; - result->fragile_left = false; - result->fragile_right = false; - result->has_changes = false; - result->has_external_tokens = has_external_tokens; - result->is_missing = false; - result->is_keyword = is_keyword; + unsigned additional_bytes_scanned = bytes_scanned - size.bytes - padding.bytes; + bool extra = symbol == ts_builtin_sym_end; + + bool is_inline = ( + symbol <= UINT8_MAX && + additional_bytes_scanned < 16 && + !has_external_tokens && + ts_subtree_can_inline(padding, size) + ); + + if (is_inline) { + return (Subtree) {{ + .parse_state = parse_state, + .symbol = symbol, + .padding_bytes = padding.bytes, + .padding_rows = padding.extent.row, + .padding_columns = padding.extent.column, + .size_bytes = size.bytes, + .additional_bytes_scanned = additional_bytes_scanned, + .visible = metadata.visible, + .named = metadata.named, + .extra = extra, + .has_changes = false, + .is_missing = false, + .is_keyword = is_keyword, + .is_inline = true, + }}; + } else { + SubtreeHeapData *data = ts_subtree_pool_allocate(pool); + *data = (SubtreeHeapData) { + .ref_count = 1, + .padding = padding, + .size = size, + .bytes_scanned = bytes_scanned, + .error_cost = 0, + .child_count = 0, + .symbol = symbol, + .parse_state = parse_state, + .visible = metadata.visible, + .named = metadata.named, + .extra = extra, + .fragile_left = false, + .fragile_right = false, + .has_changes = false, + .has_external_tokens = has_external_tokens, + .is_missing = false, + .is_keyword = is_keyword, + .first_leaf = {.symbol = 0, .parse_state = 0}, + }; + return (Subtree) {.ptr = data}; + } +} + +Subtree ts_subtree_new_error( + SubtreePool *pool, int32_t lookahead_char, Length padding, Length size, + uint32_t bytes_scanned, TSStateId parse_state, const TSLanguage *language +) { + Subtree result = ts_subtree_new_leaf( + pool, ts_builtin_sym_error, padding, size, bytes_scanned, + parse_state, false, false, language + ); + SubtreeHeapData *data = (SubtreeHeapData *)result.ptr; + data->fragile_left = true; + data->fragile_right = true; + data->lookahead_char = lookahead_char; return result; } -Subtree *ts_subtree__new(SubtreePool *pool, TSSymbol symbol, Length padding, Length size, - bool is_small, const TSLanguage *language) { - TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); - Subtree *result = ts_subtree_pool_allocate(pool, is_small); - result->padding = padding; - result->size = size; - result->ref_count = 1; - result->bytes_scanned = 0; - result->error_cost = 0; - result->child_count = 0; - result->is_small = is_small; - result->visible = metadata.visible; - result->named = metadata.named; - result->extra = symbol == ts_builtin_sym_end; - result->fragile_left = false; - result->fragile_right = false; - result->has_changes = false; - result->has_external_tokens = false; - result->is_missing = false; - result->is_keyword = false; - result->symbol = symbol; - result->parse_state = 0; - return result; -} +MutableSubtree ts_subtree_make_mut(SubtreePool *pool, Subtree self) { + if (self.data.is_inline) return (MutableSubtree) {self.data}; + if (self.ptr->ref_count == 1) return ts_subtree_to_mut_unsafe(self); -Subtree *ts_subtree_new_error(SubtreePool *pool, Length size, Length padding, - int32_t lookahead_char, const TSLanguage *language) { - Subtree *result = ts_subtree__new(pool, ts_builtin_sym_error, padding, size, false, language); - result->fragile_left = true; - result->fragile_right = true; - result->lookahead_char = lookahead_char; - return result; -} - -Subtree *ts_subtree_new_copy(SubtreePool *pool, const Subtree *self) { - Subtree *result = ts_subtree_pool_allocate(pool, self->is_small); - memcpy(result, self, self->is_small ? SMALL_TREE_SIZE : LARGE_TREE_SIZE); + SubtreeHeapData *result = ts_subtree_pool_allocate(pool); + memcpy(result, self.ptr, sizeof(SubtreeHeapData)); if (result->child_count > 0) { - result->children = ts_calloc(self->child_count, sizeof(const Subtree *)); - memcpy(result->children, self->children, self->child_count * sizeof(const Subtree *)); + result->children = ts_calloc(self.ptr->child_count, sizeof(Subtree)); + memcpy(result->children, self.ptr->children, result->child_count * sizeof(Subtree)); for (uint32_t i = 0; i < result->child_count; i++) { ts_subtree_retain(result->children[i]); } } else if (result->has_external_tokens) { - result->external_scanner_state = ts_external_scanner_state_copy(&self->external_scanner_state); + result->external_scanner_state = ts_external_scanner_state_copy(&self.ptr->external_scanner_state); } result->ref_count = 1; - return result; + ts_subtree_release(pool, self); + return (MutableSubtree) {.ptr = result}; } -Subtree *ts_subtree_make_mut(SubtreePool *pool, const Subtree *self) { - if (self->ref_count == 1) { - return (Subtree *)self; - } else { - Subtree *result = ts_subtree_new_copy(pool, self); - ts_subtree_release(pool, self); - return result; - } -} - -static void ts_subtree__compress(Subtree *self, unsigned count, const TSLanguage *language, +static void ts_subtree__compress(MutableSubtree self, unsigned count, const TSLanguage *language, MutableSubtreeArray *stack) { unsigned initial_stack_size = stack->size; - Subtree *tree = self; + MutableSubtree tree = self; + TSSymbol symbol = tree.ptr->symbol; for (unsigned i = 0; i < count; i++) { - if (tree->ref_count > 1 || tree->child_count != 2) break; + if (tree.ptr->ref_count > 1 || tree.ptr->child_count != 2) break; - Subtree *child = (Subtree *)tree->children[0]; + MutableSubtree child = ts_subtree_to_mut_unsafe(tree.ptr->children[0]); if ( - child->ref_count > 1 || - child->child_count != 2 || - child->symbol != tree->symbol + child.data.is_inline || + child.ptr->child_count != 2 || + child.ptr->ref_count > 1 || + child.ptr->symbol != symbol ) break; - Subtree *grandchild = (Subtree *)child->children[0]; + MutableSubtree grandchild = ts_subtree_to_mut_unsafe(child.ptr->children[0]); if ( - grandchild->ref_count > 1 || - grandchild->child_count != 2 || - grandchild->symbol != tree->symbol + grandchild.data.is_inline || + grandchild.ptr->child_count != 2 || + grandchild.ptr->ref_count > 1 || + grandchild.ptr->symbol != symbol ) break; - tree->children[0] = grandchild; - child->children[0] = grandchild->children[1]; - grandchild->children[1] = child; + tree.ptr->children[0] = ts_subtree_from_mut(grandchild); + child.ptr->children[0] = grandchild.ptr->children[1]; + grandchild.ptr->children[1] = ts_subtree_from_mut(child); array_push(stack, tree); tree = grandchild; } while (stack->size > initial_stack_size) { tree = array_pop(stack); - assert(tree); - Subtree *child = (Subtree *)tree->children[0]; - Subtree *grandchild = (Subtree *)child->children[1]; - ts_subtree_set_children(grandchild, grandchild->children, grandchild->child_count, language); - ts_subtree_set_children(child, child->children, child->child_count, language); - ts_subtree_set_children(tree, tree->children, tree->child_count, language); + MutableSubtree child = ts_subtree_to_mut_unsafe(tree.ptr->children[0]); + MutableSubtree grandchild = ts_subtree_to_mut_unsafe(child.ptr->children[1]); + ts_subtree_set_children(grandchild, grandchild.ptr->children, grandchild.ptr->child_count, language); + ts_subtree_set_children(child, child.ptr->children, child.ptr->child_count, language); + ts_subtree_set_children(tree, tree.ptr->children, tree.ptr->child_count, language); } } -void ts_subtree_balance(const Subtree *self, SubtreePool *pool, const TSLanguage *language) { +void ts_subtree_balance(Subtree self, SubtreePool *pool, const TSLanguage *language) { array_clear(&pool->tree_stack); - if (self->ref_count == 1) { - array_push(&pool->tree_stack, (Subtree *)self); + if (ts_subtree_child_count(self) > 0 && self.ptr->ref_count == 1) { + array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(self)); } while (pool->tree_stack.size > 0) { - Subtree *tree = array_pop(&pool->tree_stack); - assert(tree); + MutableSubtree tree = array_pop(&pool->tree_stack); - if ( - tree->child_count > 0 && - tree->repeat_depth > 0 && - tree->children[0]->child_count > 0 && - tree->children[1]->child_count > 0 && - tree->children[0]->repeat_depth > tree->children[1]->repeat_depth - ) { - unsigned n = tree->children[0]->repeat_depth - tree->children[1]->repeat_depth; - for (unsigned i = n / 2; i > 0; i /= 2) { - ts_subtree__compress(tree, i, language, &pool->tree_stack); - n -= i; + if (tree.ptr->repeat_depth > 0) { + Subtree child1 = tree.ptr->children[0]; + Subtree child2 = tree.ptr->children[1]; + if ( + ts_subtree_child_count(child1) > 0 && + ts_subtree_child_count(child2) > 0 && + child1.ptr->repeat_depth > child2.ptr->repeat_depth + ) { + unsigned n = child1.ptr->repeat_depth - child2.ptr->repeat_depth; + for (unsigned i = n / 2; i > 0; i /= 2) { + ts_subtree__compress(tree, i, language, &pool->tree_stack); + n -= i; + } } } - for (uint32_t i = 0; i < tree->child_count; i++) { - const Subtree *child = tree->children[i]; - if (child->ref_count == 1) { - array_push(&pool->tree_stack, (Subtree *)child); + for (uint32_t i = 0; i < tree.ptr->child_count; i++) { + Subtree child = tree.ptr->children[i]; + if (ts_subtree_child_count(child) > 0 && child.ptr->ref_count == 1) { + array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(child)); } } } } -void ts_subtree_set_children(Subtree *self, const Subtree **children, uint32_t child_count, const TSLanguage *language) { - if (self->child_count > 0 && children != self->children) { - ts_free(self->children); +void ts_subtree_set_children( + MutableSubtree self, Subtree *children, uint32_t child_count, const TSLanguage *language +) { + assert(!self.data.is_inline); + + if (self.ptr->child_count > 0 && children != self.ptr->children) { + ts_free(self.ptr->children); } - assert(!self->is_small); - - self->child_count = child_count; - self->children = children; - self->named_child_count = 0; - self->visible_child_count = 0; - self->error_cost = 0; - self->repeat_depth = 0; - self->node_count = 1; - self->has_external_tokens = false; - self->dynamic_precedence = 0; + self.ptr->child_count = child_count; + self.ptr->children = children; + self.ptr->named_child_count = 0; + self.ptr->visible_child_count = 0; + self.ptr->error_cost = 0; + self.ptr->repeat_depth = 0; + self.ptr->node_count = 1; + self.ptr->has_external_tokens = false; + self.ptr->dynamic_precedence = 0; uint32_t non_extra_index = 0; - const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); + const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self.ptr->alias_sequence_id); - for (uint32_t i = 0; i < self->child_count; i++) { - const Subtree *child = self->children[i]; + for (uint32_t i = 0; i < self.ptr->child_count; i++) { + Subtree child = self.ptr->children[i]; if (i == 0) { - self->padding = child->padding; - self->size = child->size; - self->bytes_scanned = child->bytes_scanned; + self.ptr->padding = ts_subtree_padding(child); + self.ptr->size = ts_subtree_size(child); + self.ptr->bytes_scanned = ts_subtree_bytes_scanned(child); } else { - uint32_t bytes_scanned = ts_subtree_total_bytes(self) + child->bytes_scanned; - if (bytes_scanned > self->bytes_scanned) self->bytes_scanned = bytes_scanned; - self->size = length_add(self->size, ts_subtree_total_size(child)); + uint32_t bytes_scanned = self.ptr->padding.bytes + self.ptr->size.bytes + ts_subtree_bytes_scanned(child); + if (bytes_scanned > self.ptr->bytes_scanned) self.ptr->bytes_scanned = bytes_scanned; + self.ptr->size = length_add(self.ptr->size, ts_subtree_total_size(child)); } - if (child->symbol != ts_builtin_sym_error_repeat) { - self->error_cost += child->error_cost; + if (ts_subtree_symbol(child) != ts_builtin_sym_error_repeat) { + self.ptr->error_cost += ts_subtree_error_cost(child); } - if (child->child_count > 0) { - self->dynamic_precedence += child->dynamic_precedence; - self->node_count += child->node_count; - } else { - self->node_count++; - } + self.ptr->dynamic_precedence += ts_subtree_dynamic_precedence(child); + self.ptr->node_count += ts_subtree_node_count(child); - if (alias_sequence && alias_sequence[non_extra_index] != 0 && !child->extra) { - self->visible_child_count++; + if (alias_sequence && alias_sequence[non_extra_index] != 0 && !ts_subtree_extra(child)) { + self.ptr->visible_child_count++; if (ts_language_symbol_metadata(language, alias_sequence[non_extra_index]).named) { - self->named_child_count++; + self.ptr->named_child_count++; } - } else if (child->visible) { - self->visible_child_count++; - if (child->named) self->named_child_count++; - } else if (child->child_count > 0) { - self->visible_child_count += child->visible_child_count; - self->named_child_count += child->named_child_count; + } else if (ts_subtree_visible(child)) { + self.ptr->visible_child_count++; + if (ts_subtree_named(child)) self.ptr->named_child_count++; + } else if (ts_subtree_child_count(child) > 0) { + self.ptr->visible_child_count += child.ptr->visible_child_count; + self.ptr->named_child_count += child.ptr->named_child_count; } - if (child->has_external_tokens) self->has_external_tokens = true; + if (ts_subtree_has_external_tokens(child)) self.ptr->has_external_tokens = true; - if (child->symbol == ts_builtin_sym_error) { - self->fragile_left = self->fragile_right = true; - self->parse_state = TS_TREE_STATE_NONE; + if (ts_subtree_is_error(child)) { + self.ptr->fragile_left = self.ptr->fragile_right = true; + self.ptr->parse_state = TS_TREE_STATE_NONE; } - if (!child->extra) non_extra_index++; + if (!ts_subtree_extra(child)) non_extra_index++; } - if (self->symbol == ts_builtin_sym_error || self->symbol == ts_builtin_sym_error_repeat) { - self->error_cost += ERROR_COST_PER_RECOVERY + - ERROR_COST_PER_SKIPPED_CHAR * self->size.bytes + - ERROR_COST_PER_SKIPPED_LINE * self->size.extent.row; - for (uint32_t i = 0; i < self->child_count; i++) { - const Subtree *child = self->children[i]; - if (child->extra) continue; - if (child->symbol == ts_builtin_sym_error && child->child_count == 0) continue; - if (child->visible) { - self->error_cost += ERROR_COST_PER_SKIPPED_TREE; - } else { - self->error_cost += ERROR_COST_PER_SKIPPED_TREE * child->visible_child_count; + if (self.ptr->symbol == ts_builtin_sym_error || self.ptr->symbol == ts_builtin_sym_error_repeat) { + self.ptr->error_cost += + ERROR_COST_PER_RECOVERY + + ERROR_COST_PER_SKIPPED_CHAR * self.ptr->size.bytes + + ERROR_COST_PER_SKIPPED_LINE * self.ptr->size.extent.row; + for (uint32_t i = 0; i < self.ptr->child_count; i++) { + Subtree child = self.ptr->children[i]; + uint32_t grandchild_count = ts_subtree_child_count(child); + if (ts_subtree_extra(child)) continue; + if (ts_subtree_is_error(child) && grandchild_count == 0) continue; + if (ts_subtree_visible(child)) { + self.ptr->error_cost += ERROR_COST_PER_SKIPPED_TREE; + } else if (grandchild_count > 0) { + self.ptr->error_cost += ERROR_COST_PER_SKIPPED_TREE * child.ptr->visible_child_count; } } } - if (self->child_count > 0) { - const Subtree *first_child = self->children[0]; - const Subtree *last_child = self->children[self->child_count - 1]; - if (first_child->child_count > 0) { - self->first_leaf = first_child->first_leaf; - } else { - self->first_leaf.symbol = first_child->symbol; - self->first_leaf.parse_state = first_child->parse_state; - } - if (first_child->fragile_left) self->fragile_left = true; - if (last_child->fragile_right) self->fragile_right = true; + if (self.ptr->child_count > 0) { + Subtree first_child = self.ptr->children[0]; + Subtree last_child = self.ptr->children[self.ptr->child_count - 1]; + + self.ptr->first_leaf.symbol = ts_subtree_leaf_symbol(first_child); + self.ptr->first_leaf.parse_state = ts_subtree_leaf_parse_state(first_child); + + if (ts_subtree_fragile_left(first_child)) self.ptr->fragile_left = true; + if (ts_subtree_fragile_right(last_child)) self.ptr->fragile_right = true; + if ( - self->child_count == 2 && - !self->visible && !self->named && - first_child->symbol == self->symbol && - last_child->symbol == self->symbol + self.ptr->child_count == 2 && + !self.ptr->visible && !self.ptr->named && + ts_subtree_symbol(first_child) == self.ptr->symbol && + ts_subtree_symbol(last_child) == self.ptr->symbol ) { - if (first_child->repeat_depth > last_child->repeat_depth) { - self->repeat_depth = first_child->repeat_depth + 1; + if (first_child.ptr->repeat_depth > last_child.ptr->repeat_depth) { + self.ptr->repeat_depth = first_child.ptr->repeat_depth + 1; } else { - self->repeat_depth = last_child->repeat_depth + 1; + self.ptr->repeat_depth = last_child.ptr->repeat_depth + 1; } } } } -Subtree *ts_subtree_new_node(SubtreePool *pool, TSSymbol symbol, SubtreeArray *children, - unsigned alias_sequence_id, const TSLanguage *language) { +MutableSubtree ts_subtree_new_node(SubtreePool *pool, TSSymbol symbol, + SubtreeArray *children, unsigned alias_sequence_id, + const TSLanguage *language) { TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); - Subtree *result = ts_subtree_pool_allocate(pool, false); - *result = (Subtree){ + bool fragile = symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat; + SubtreeHeapData *data = ts_subtree_pool_allocate(pool); + *data = (SubtreeHeapData) { .ref_count = 1, .symbol = symbol, .alias_sequence_id = alias_sequence_id, .visible = metadata.visible, .named = metadata.named, .has_changes = false, + .fragile_left = fragile, + .fragile_right = fragile, .is_keyword = false, .node_count = 0, + .first_leaf = {.symbol = 0, .parse_state = 0}, }; - if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { - result->fragile_left = true; - result->fragile_right = true; - } + MutableSubtree result = {.ptr = data}; ts_subtree_set_children(result, children->contents, children->size, language); return result; } -Subtree *ts_subtree_new_error_node(SubtreePool *pool, SubtreeArray *children, - const TSLanguage *language) { - return ts_subtree_new_node(pool, ts_builtin_sym_error, children, 0, language); +Subtree ts_subtree_new_error_node(SubtreePool *pool, SubtreeArray *children, + bool extra, const TSLanguage *language) { + MutableSubtree result = ts_subtree_new_node( + pool, ts_builtin_sym_error, children, 0, language + ); + result.ptr->extra = extra; + return ts_subtree_from_mut(result); } -Subtree *ts_subtree_new_missing_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, - const TSLanguage *language) { - Subtree *result = ts_subtree__new(pool, symbol, padding, length_zero(), true, language); - result->is_missing = true; - result->error_cost = ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY; +Subtree ts_subtree_new_missing_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, + const TSLanguage *language) { + Subtree result = ts_subtree_new_leaf( + pool, symbol, padding, length_zero(), 0, + 0, false, false, language + ); + + if (result.data.is_inline) { + result.data.is_missing = true; + } else { + ((SubtreeHeapData *)result.ptr)->is_missing = true; + } + return result; } -void ts_subtree_retain(const Subtree *self) { - assert(self->ref_count > 0); - atomic_inc((volatile uint32_t *)&self->ref_count); - assert(self->ref_count != 0); +void ts_subtree_retain(Subtree self) { + if (self.data.is_inline) return; + assert(self.ptr->ref_count > 0); + atomic_inc((volatile uint32_t *)&self.ptr->ref_count); + assert(self.ptr->ref_count != 0); } -void ts_subtree_release(SubtreePool *pool, const Subtree *self) { +void ts_subtree_release(SubtreePool *pool, Subtree self) { + if (self.data.is_inline) return; array_clear(&pool->tree_stack); - assert(self->ref_count > 0); - if (atomic_dec((volatile uint32_t *)&self->ref_count) == 0) { - array_push(&pool->tree_stack, (Subtree *)self); + assert(self.ptr->ref_count > 0); + if (atomic_dec((volatile uint32_t *)&self.ptr->ref_count) == 0) { + array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(self)); } while (pool->tree_stack.size > 0) { - Subtree *tree = array_pop(&pool->tree_stack); - if (tree->child_count > 0) { - for (uint32_t i = 0; i < tree->child_count; i++) { - const Subtree *child = tree->children[i]; - if (atomic_dec((volatile uint32_t *)&child->ref_count) == 0) { - array_push(&pool->tree_stack, (Subtree *)child); + MutableSubtree tree = array_pop(&pool->tree_stack); + if (tree.ptr->child_count > 0) { + for (uint32_t i = 0; i < tree.ptr->child_count; i++) { + Subtree child = tree.ptr->children[i]; + if (child.data.is_inline) continue; + assert(child.ptr->ref_count > 0); + if (atomic_dec((volatile uint32_t *)&child.ptr->ref_count) == 0) { + array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(child)); } } - ts_free(tree->children); - } else if (tree->has_external_tokens) { - ts_external_scanner_state_delete(&tree->external_scanner_state); + ts_free(tree.ptr->children); + } else if (tree.ptr->has_external_tokens) { + ts_external_scanner_state_delete(&tree.ptr->external_scanner_state); } - ts_subtree_pool_free(pool, tree); + ts_subtree_pool_free(pool, tree.ptr); } } -bool ts_subtree_eq(const Subtree *self, const Subtree *other) { - if (self) { - if (!other) return false; - } else { - return !other; +bool ts_subtree_eq(Subtree self, Subtree other) { + if (self.data.is_inline || other.data.is_inline) { + return memcmp(&self, &other, sizeof(SubtreeInlineData)) == 0; } - if (self->symbol != other->symbol) return false; - if (self->visible != other->visible) return false; - if (self->named != other->named) return false; - if (self->padding.bytes != other->padding.bytes) return false; - if (self->size.bytes != other->size.bytes) return false; - if (self->symbol == ts_builtin_sym_error) return self->lookahead_char == other->lookahead_char; - if (self->child_count != other->child_count) return false; - if (self->child_count > 0) { - if (self->visible_child_count != other->visible_child_count) return false; - if (self->named_child_count != other->named_child_count) return false; + if (self.ptr) { + if (!other.ptr) return false; + } else { + return !other.ptr; + } - for (uint32_t i = 0; i < self->child_count; i++) { - if (!ts_subtree_eq(self->children[i], other->children[i])) { + if (self.ptr->symbol != other.ptr->symbol) return false; + if (self.ptr->visible != other.ptr->visible) return false; + if (self.ptr->named != other.ptr->named) return false; + if (self.ptr->padding.bytes != other.ptr->padding.bytes) return false; + if (self.ptr->size.bytes != other.ptr->size.bytes) return false; + if (self.ptr->symbol == ts_builtin_sym_error) return self.ptr->lookahead_char == other.ptr->lookahead_char; + if (self.ptr->child_count != other.ptr->child_count) return false; + if (self.ptr->child_count > 0) { + if (self.ptr->visible_child_count != other.ptr->visible_child_count) return false; + if (self.ptr->named_child_count != other.ptr->named_child_count) return false; + + for (uint32_t i = 0; i < self.ptr->child_count; i++) { + if (!ts_subtree_eq(self.ptr->children[i], other.ptr->children[i])) { return false; } } @@ -526,33 +560,34 @@ bool ts_subtree_eq(const Subtree *self, const Subtree *other) { return true; } -int ts_subtree_compare(const Subtree *left, const Subtree *right) { - if (left->symbol < right->symbol) - return -1; - if (right->symbol < left->symbol) - return 1; - if (left->child_count < right->child_count) - return -1; - if (right->child_count < left->child_count) - return 1; - for (uint32_t i = 0; i < left->child_count; i++) { - const Subtree *left_child = left->children[i]; - const Subtree *right_child = right->children[i]; +int ts_subtree_compare(Subtree left, Subtree right) { + if (ts_subtree_symbol(left) < ts_subtree_symbol(right)) return -1; + if (ts_subtree_symbol(right) < ts_subtree_symbol(left)) return 1; + if (ts_subtree_child_count(left) < ts_subtree_child_count(right)) return -1; + if (ts_subtree_child_count(right) < ts_subtree_child_count(left)) return 1; + for (uint32_t i = 0, n = ts_subtree_child_count(left); i < n; i++) { + Subtree left_child = left.ptr->children[i]; + Subtree right_child = right.ptr->children[i]; switch (ts_subtree_compare(left_child, right_child)) { - case -1: - return -1; - case 1: - return 1; - default: - break; + case -1: return -1; + case 1: return 1; + default: break; } } return 0; } -const Subtree *ts_subtree_edit(const Subtree *self, const TSInputEdit *edit, SubtreePool *pool) { +static inline void ts_subtree_set_has_changes(MutableSubtree *self) { + if (self->data.is_inline) { + self->data.has_changes = true; + } else { + self->ptr->has_changes = true; + } +} + +Subtree ts_subtree_edit(Subtree self, const TSInputEdit *edit, SubtreePool *pool) { typedef struct { - const Subtree **tree; + Subtree *tree; Edit edit; } StackEntry; @@ -571,47 +606,83 @@ const Subtree *ts_subtree_edit(const Subtree *self, const TSInputEdit *edit, Sub Edit edit = entry.edit; bool is_noop = edit.old_end.bytes == edit.start.bytes && edit.new_end.bytes == edit.start.bytes; bool is_pure_insertion = edit.old_end.bytes == edit.start.bytes; - if (is_noop && edit.start.bytes >= (*entry.tree)->bytes_scanned) continue; + uint32_t bytes_scanned = ts_subtree_bytes_scanned(*entry.tree); + if (is_noop && edit.start.bytes >= bytes_scanned) continue; - Subtree *result = ts_subtree_make_mut(pool, *entry.tree); - *entry.tree = result; + Length size = ts_subtree_size(*entry.tree); + Length padding = ts_subtree_padding(*entry.tree); // If the edit is entirely within the space before this subtree, then shift this // subtree over according to the edit without changing its size. - if (edit.old_end.bytes <= result->padding.bytes) { - result->padding = length_add(edit.new_end, length_sub(result->padding, edit.old_end)); + if (edit.old_end.bytes <= padding.bytes) { + padding = length_add(edit.new_end, length_sub(padding, edit.old_end)); } // If the edit starts in the space before this subtree and extends into this subtree, // shrink the subtree's content to compensate for the change in the space before it. - else if (edit.start.bytes < result->padding.bytes) { - result->size = length_sub(result->size, length_sub(edit.old_end, result->padding)); - result->padding = edit.new_end; + else if (edit.start.bytes < padding.bytes) { + size = length_sub(size, length_sub(edit.old_end, padding)); + padding = edit.new_end; } // If the edit is a pure insertion right at the start of the subtree, // shift the subtree over according to the insertion. - else if (edit.start.bytes == result->padding.bytes && is_pure_insertion) { - result->padding = edit.new_end; + else if (edit.start.bytes == padding.bytes && is_pure_insertion) { + padding = edit.new_end; } // If the edit is within this subtree, resize the subtree to reflect the edit. else { - uint32_t total_bytes = ts_subtree_total_bytes(*entry.tree); + uint32_t total_bytes = padding.bytes + size.bytes; if (edit.start.bytes < total_bytes || (edit.start.bytes == total_bytes && is_pure_insertion)) { - result->size = length_add( - length_sub(edit.new_end, result->padding), - length_sub(result->size, length_sub(edit.old_end, result->padding)) + size = length_add( + length_sub(edit.new_end, padding), + length_sub(size, length_sub(edit.old_end, padding)) ); } } - result->has_changes = true; + MutableSubtree result = ts_subtree_make_mut(pool, *entry.tree); + + if (result.data.is_inline) { + if (ts_subtree_can_inline(padding, size)) { + result.data.padding_bytes = padding.bytes; + result.data.padding_rows = padding.extent.row; + result.data.padding_columns = padding.extent.column; + result.data.size_bytes = size.bytes; + } else { + SubtreeHeapData *data = ts_subtree_pool_allocate(pool); + data->ref_count = 1; + data->padding = padding; + data->size = size; + data->bytes_scanned = bytes_scanned; + data->error_cost = 0; + data->child_count = 0; + data->symbol = result.data.symbol; + data->parse_state = result.data.parse_state; + data->visible = result.data.visible; + data->named = result.data.named; + data->extra = result.data.extra; + data->fragile_left = false; + data->fragile_right = false; + data->has_changes = false; + data->has_external_tokens = false; + data->is_missing = result.data.is_missing; + data->is_keyword = result.data.is_keyword; + result.ptr = data; + } + } else { + result.ptr->padding = padding; + result.ptr->size = size; + } + + ts_subtree_set_has_changes(&result); + *entry.tree = ts_subtree_from_mut(result); Length child_left, child_right = length_zero(); - for (uint32_t i = 0; i < result->child_count; i++) { - const Subtree **child = &result->children[i]; + for (uint32_t i = 0, n = ts_subtree_child_count(*entry.tree); i < n; i++) { + Subtree *child = &result.ptr->children[i]; Length child_size = ts_subtree_total_size(*child); child_left = child_right; child_right = length_add(child_left, child_size); @@ -659,12 +730,12 @@ const Subtree *ts_subtree_edit(const Subtree *self, const TSInputEdit *edit, Sub return self; } -const Subtree *ts_subtree_last_external_token(const Subtree *tree) { - if (!tree->has_external_tokens) return NULL; - while (tree->child_count > 0) { - for (uint32_t i = tree->child_count - 1; i + 1 > 0; i--) { - const Subtree *child = tree->children[i]; - if (child->has_external_tokens) { +Subtree ts_subtree_last_external_token(Subtree tree) { + if (!ts_subtree_has_external_tokens(tree)) return NULL_SUBTREE; + while (tree.ptr->child_count > 0) { + for (uint32_t i = tree.ptr->child_count - 1; i + 1 > 0; i--) { + Subtree child = tree.ptr->children[i]; + if (ts_subtree_has_external_tokens(child)) { tree = child; break; } @@ -690,19 +761,19 @@ static size_t ts_subtree__write_char_to_string(char *s, size_t n, int32_t c) { return snprintf(s, n, "%d", c); } -static size_t ts_subtree__write_to_string(const Subtree *self, char *string, size_t limit, - const TSLanguage *language, bool is_root, - bool include_all, TSSymbol alias_symbol, - bool alias_is_named) { - if (!self) return snprintf(string, limit, "(NULL)"); +static size_t ts_subtree__write_to_string(Subtree self, char *string, size_t limit, + const TSLanguage *language, bool is_root, + bool include_all, TSSymbol alias_symbol, + bool alias_is_named) { + if (!self.ptr) return snprintf(string, limit, "(NULL)"); char *cursor = string; char **writer = (limit > 0) ? &cursor : &string; bool visible = include_all || is_root || - self->is_missing || - (self->visible && self->named) || + ts_subtree_missing(self) || + (ts_subtree_visible(self) && ts_subtree_named(self)) || alias_is_named; if (visible && !is_root) { @@ -710,24 +781,24 @@ static size_t ts_subtree__write_to_string(const Subtree *self, char *string, siz } if (visible) { - if (self->symbol == ts_builtin_sym_error && self->child_count == 0 && self->size.bytes > 0) { + if (ts_subtree_is_error(self) && ts_subtree_child_count(self) == 0 && self.ptr->size.bytes > 0) { cursor += snprintf(*writer, limit, "(UNEXPECTED "); - cursor += ts_subtree__write_char_to_string(*writer, limit, self->lookahead_char); - } else if (self->is_missing) { + cursor += ts_subtree__write_char_to_string(*writer, limit, self.ptr->lookahead_char); + } else if (ts_subtree_missing(self)) { cursor += snprintf(*writer, limit, "(MISSING"); } else { - TSSymbol symbol = alias_symbol ? alias_symbol : self->symbol; + TSSymbol symbol = alias_symbol ? alias_symbol : ts_subtree_symbol(self); const char *symbol_name = ts_language_symbol_name(language, symbol); cursor += snprintf(*writer, limit, "(%s", symbol_name); } } - if (self->child_count) { - const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); + if (ts_subtree_child_count(self)) { + const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self.ptr->alias_sequence_id); uint32_t structural_child_index = 0; - for (uint32_t i = 0; i < self->child_count; i++) { - const Subtree *child = self->children[i]; - if (child->extra) { + for (uint32_t i = 0; i < self.ptr->child_count; i++) { + Subtree child = self.ptr->children[i]; + if (ts_subtree_extra(child)) { cursor += ts_subtree__write_to_string( child, *writer, limit, language, false, include_all, @@ -751,7 +822,11 @@ static size_t ts_subtree__write_to_string(const Subtree *self, char *string, siz return cursor - string; } -char *ts_subtree_string(const Subtree *self, const TSLanguage *language, bool include_all) { +static inline uint32_t ts_subtree_repeat_depth(Subtree self) { + return ts_subtree_child_count(self) ? self.ptr->repeat_depth : 0; +} + +char *ts_subtree_string(Subtree self, const TSLanguage *language, bool include_all) { char scratch_string[1]; size_t size = ts_subtree__write_to_string( self, scratch_string, 0, @@ -763,62 +838,73 @@ char *ts_subtree_string(const Subtree *self, const TSLanguage *language, bool in return result; } -void ts_subtree__print_dot_graph(const Subtree *self, uint32_t byte_offset, - const TSLanguage *language, TSSymbol alias_symbol, FILE *f) { - TSSymbol symbol = alias_symbol ? alias_symbol : self->symbol; - fprintf(f, "tree_%p [label=\"%s\"", self, ts_language_symbol_name(language, symbol)); +void ts_subtree__print_dot_graph(Subtree self, uint32_t start_offset, + const TSLanguage *language, TSSymbol alias_symbol, + FILE *f) { + TSSymbol subtree_symbol = ts_subtree_symbol(self); + TSSymbol symbol = alias_symbol ? alias_symbol : subtree_symbol; + uint32_t end_offset = start_offset + ts_subtree_total_bytes(self); + fprintf( + f, "tree_%u_%u_%u [label=\"%s\"", + start_offset, end_offset, subtree_symbol, ts_language_symbol_name(language, symbol) + ); - if (self->child_count == 0) - fprintf(f, ", shape=plaintext"); - if (self->extra) - fprintf(f, ", fontcolor=gray"); + if (ts_subtree_child_count(self) == 0) fprintf(f, ", shape=plaintext"); + if (ts_subtree_extra(self)) fprintf(f, ", fontcolor=gray"); fprintf(f, ", tooltip=\"" - "address:%p\n" "range:%u - %u\n" "state:%d\n" "error-cost:%u\n" "repeat-depth:%u\n" "bytes-scanned:%u\"]\n", - self, - byte_offset, byte_offset + ts_subtree_total_bytes(self), - self->parse_state, - self->error_cost, - self->child_count > 0 ? self->repeat_depth : 0, - self->bytes_scanned + start_offset, end_offset, + ts_subtree_parse_state(self), + ts_subtree_error_cost(self), + ts_subtree_repeat_depth(self), + ts_subtree_bytes_scanned(self) ); - const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); + uint32_t child_start_offset = start_offset; uint32_t structural_child_index = 0; - for (uint32_t i = 0; i < self->child_count; i++) { - const Subtree *child = self->children[i]; - if (child->extra) { - ts_subtree__print_dot_graph(child, byte_offset, language, 0, f); + const TSSymbol *alias_sequence = ts_language_alias_sequence( + language, + ts_subtree_alias_sequence_id(self) + ); + for (uint32_t i = 0, n = ts_subtree_child_count(self); i < n; i++) { + Subtree child = self.ptr->children[i]; + uint32_t child_end_offset = child_start_offset + ts_subtree_total_bytes(child); + if (ts_subtree_extra(child)) { + ts_subtree__print_dot_graph(child, child_start_offset, language, 0, f); } else { TSSymbol alias_symbol = alias_sequence ? alias_sequence[structural_child_index] : 0; - ts_subtree__print_dot_graph(child, byte_offset, language, alias_symbol, f); + ts_subtree__print_dot_graph(child, child_start_offset, language, alias_symbol, f); structural_child_index++; } - fprintf(f, "tree_%p -> tree_%p [tooltip=%u]\n", self, child, i); - byte_offset += ts_subtree_total_bytes(child); + fprintf( + f, "tree_%u_%u_%u -> tree_%u_%u_%u [tooltip=%u]\n", + start_offset, end_offset, subtree_symbol, + child_start_offset, child_end_offset, ts_subtree_symbol(child), i + ); + child_start_offset = child_end_offset; } } -void ts_subtree_print_dot_graph(const Subtree *self, const TSLanguage *language, FILE *f) { +void ts_subtree_print_dot_graph(Subtree self, const TSLanguage *language, FILE *f) { fprintf(f, "digraph tree {\n"); fprintf(f, "edge [arrowhead=none]\n"); ts_subtree__print_dot_graph(self, 0, language, 0, f); fprintf(f, "}\n"); } -bool ts_subtree_external_scanner_state_eq(const Subtree *self, const Subtree *other) { +bool ts_subtree_external_scanner_state_eq(Subtree self, Subtree other) { const ExternalScannerState *state1 = &empty_state; const ExternalScannerState *state2 = &empty_state; - if (self && !self->child_count && self->has_external_tokens) { - state1 = &self->external_scanner_state; + if (self.ptr && ts_subtree_has_external_tokens(self) && !self.ptr->child_count) { + state1 = &self.ptr->external_scanner_state; } - if (other && !other->child_count && other->has_external_tokens) { - state2 = &other->external_scanner_state; + if (other.ptr && ts_subtree_has_external_tokens(other) && !other.ptr->child_count) { + state2 = &other.ptr->external_scanner_state; } return ts_external_scanner_state_eq(state1, state2); } diff --git a/src/runtime/subtree.h b/src/runtime/subtree.h index 52b5c2ff..349ab85e 100644 --- a/src/runtime/subtree.h +++ b/src/runtime/subtree.h @@ -9,10 +9,15 @@ extern "C" { #include #include "runtime/length.h" #include "runtime/array.h" +#include "runtime/error_costs.h" #include "tree_sitter/runtime.h" #include "tree_sitter/parser.h" extern TSStateId TS_TREE_STATE_NONE; +#define NULL_SUBTREE ((Subtree) {.ptr = NULL}) + +typedef union Subtree Subtree; +typedef union MutableSubtree MutableSubtree; typedef struct { union { @@ -22,9 +27,24 @@ typedef struct { uint32_t length; } ExternalScannerState; -typedef struct Subtree Subtree; +typedef struct { + bool is_inline : 1; + bool visible : 1; + bool named : 1; + bool extra : 1; + bool has_changes : 1; + bool is_missing : 1; + bool is_keyword : 1; + uint8_t symbol; + uint8_t padding_bytes; + uint8_t size_bytes; + uint8_t padding_columns; + uint8_t padding_rows : 4; + uint8_t additional_bytes_scanned : 4; + uint16_t parse_state; +} SubtreeInlineData; -struct Subtree { +typedef struct { volatile uint32_t ref_count; Length padding; Length size; @@ -34,7 +54,6 @@ struct Subtree { TSSymbol symbol; TSStateId parse_state; - bool is_small : 1; bool visible : 1; bool named : 1; bool extra : 1; @@ -48,7 +67,7 @@ struct Subtree { union { // Non-terminal subtrees (`child_count > 0`) struct { - const Subtree **children; + Subtree *children; uint32_t visible_child_count; uint32_t named_child_count; uint32_t node_count; @@ -67,14 +86,23 @@ struct Subtree { // Error terminal subtrees (`child_count == 0 && symbol == ts_builtin_sym_error`) int32_t lookahead_char; }; +} SubtreeHeapData; + +union Subtree { + SubtreeInlineData data; + const SubtreeHeapData *ptr; }; -typedef Array(const Subtree *) SubtreeArray; -typedef Array(Subtree *) MutableSubtreeArray; +union MutableSubtree { + SubtreeInlineData data; + SubtreeHeapData *ptr; +}; + +typedef Array(Subtree) SubtreeArray; +typedef Array(MutableSubtree) MutableSubtreeArray; typedef struct { MutableSubtreeArray free_trees; - MutableSubtreeArray free_small_trees; MutableSubtreeArray tree_stack; } SubtreePool; @@ -88,41 +116,176 @@ void ts_subtree_array_reverse(SubtreeArray *); SubtreePool ts_subtree_pool_new(uint32_t capacity); void ts_subtree_pool_delete(SubtreePool *); -Subtree *ts_subtree_pool_allocate(SubtreePool *, bool); -void ts_subtree_pool_free(SubtreePool *, Subtree *); -Subtree *ts_subtree_new_leaf( +Subtree ts_subtree_new_leaf( SubtreePool *, TSSymbol, Length, Length, uint32_t, TSStateId, bool, bool, const TSLanguage * ); -Subtree *ts_subtree_new_node(SubtreePool *, TSSymbol, SubtreeArray *, unsigned, const TSLanguage *); -Subtree *ts_subtree_new_copy(SubtreePool *, const Subtree *); -Subtree *ts_subtree_new_error_node(SubtreePool *, SubtreeArray *, const TSLanguage *); -Subtree *ts_subtree_new_error(SubtreePool *, Length, Length, int32_t, const TSLanguage *); -Subtree *ts_subtree_new_missing_leaf(SubtreePool *, TSSymbol, Length, const TSLanguage *); -Subtree *ts_subtree_make_mut(SubtreePool *, const Subtree *); -void ts_subtree_retain(const Subtree *tree); -void ts_subtree_release(SubtreePool *, const Subtree *tree); -bool ts_subtree_eq(const Subtree *tree1, const Subtree *tree2); -int ts_subtree_compare(const Subtree *tree1, const Subtree *tree2); -void ts_subtree_set_children(Subtree *, const Subtree **, uint32_t, const TSLanguage *); -void ts_subtree_balance(const Subtree *, SubtreePool *, const TSLanguage *); -const Subtree *ts_subtree_edit(const Subtree *, const TSInputEdit *edit, SubtreePool *); -char *ts_subtree_string(const Subtree *, const TSLanguage *, bool include_all); -void ts_subtree_print_dot_graph(const Subtree *, const TSLanguage *, FILE *); -const Subtree *ts_subtree_last_external_token(const Subtree *); -bool ts_subtree_external_scanner_state_eq(const Subtree *, const Subtree *); +Subtree ts_subtree_new_error( + SubtreePool *, int32_t, Length, Length, uint32_t, TSStateId, const TSLanguage * +); +MutableSubtree ts_subtree_new_node(SubtreePool *, TSSymbol, SubtreeArray *, unsigned, const TSLanguage *); +Subtree ts_subtree_new_error_node(SubtreePool *, SubtreeArray *, bool, const TSLanguage *); +Subtree ts_subtree_new_missing_leaf(SubtreePool *, TSSymbol, Length, const TSLanguage *); +MutableSubtree ts_subtree_make_mut(SubtreePool *, Subtree); +void ts_subtree_retain(Subtree); +void ts_subtree_release(SubtreePool *, Subtree); +bool ts_subtree_eq(Subtree, Subtree); +int ts_subtree_compare(Subtree, Subtree); +void ts_subtree_set_children(MutableSubtree, Subtree *, uint32_t, const TSLanguage *); +void ts_subtree_balance(Subtree, SubtreePool *, const TSLanguage *); +Subtree ts_subtree_edit(Subtree, const TSInputEdit *edit, SubtreePool *); +char *ts_subtree_string(Subtree, const TSLanguage *, bool include_all); +void ts_subtree_print_dot_graph(Subtree, const TSLanguage *, FILE *); +Subtree ts_subtree_last_external_token(Subtree); +bool ts_subtree_external_scanner_state_eq(Subtree, Subtree); -static inline uint32_t ts_subtree_total_bytes(const Subtree *self) { - return self->padding.bytes + self->size.bytes; +#define SUBTREE_GET(self, name) (self.data.is_inline ? self.data.name : self.ptr->name) + +static inline TSSymbol ts_subtree_symbol(Subtree self) { return SUBTREE_GET(self, symbol); } +static inline bool ts_subtree_visible(Subtree self) { return SUBTREE_GET(self, visible); } +static inline bool ts_subtree_named(Subtree self) { return SUBTREE_GET(self, named); } +static inline bool ts_subtree_extra(Subtree self) { return SUBTREE_GET(self, extra); } +static inline bool ts_subtree_has_changes(Subtree self) { return SUBTREE_GET(self, has_changes); } +static inline bool ts_subtree_missing(Subtree self) { return SUBTREE_GET(self, is_missing); } +static inline bool ts_subtree_is_keyword(Subtree self) { return SUBTREE_GET(self, is_keyword); } +static inline TSStateId ts_subtree_parse_state(Subtree self) { return SUBTREE_GET(self, parse_state); } + +#undef SUBTREE_GET + +static inline void ts_subtree_set_symbol(MutableSubtree *self, TSSymbol symbol) { + if (self->data.is_inline) { + assert(symbol < UINT8_MAX); + self->data.symbol = symbol; + } else { + self->ptr->symbol = symbol; + } } -static inline Length ts_subtree_total_size(const Subtree *self) { - return length_add(self->padding, self->size); +static inline void ts_subtree_set_extra(MutableSubtree *self) { + if (self->data.is_inline) { + self->data.extra = true; + } else { + self->ptr->extra = true; + } } -static inline TSSymbol ts_subtree_leaf_symbol(const Subtree *self) { - return self->child_count > 0 ? self->first_leaf.symbol : self->symbol; +static inline TSSymbol ts_subtree_leaf_symbol(Subtree self) { + if (self.data.is_inline) return self.data.symbol; + if (self.ptr->child_count == 0) return self.ptr->symbol; + return self.ptr->first_leaf.symbol; +} + +static inline TSStateId ts_subtree_leaf_parse_state(Subtree self) { + if (self.data.is_inline) return self.data.parse_state; + if (self.ptr->child_count == 0) return self.ptr->parse_state; + return self.ptr->first_leaf.parse_state; +} + +static inline Length ts_subtree_padding(Subtree self) { + if (self.data.is_inline) { + Length result = {self.data.padding_bytes, {self.data.padding_rows, self.data.padding_columns}}; + return result; + } else { + return self.ptr->padding; + } +} + +static inline Length ts_subtree_size(Subtree self) { + if (self.data.is_inline) { + Length result = {self.data.size_bytes, {0, self.data.size_bytes}}; + return result; + } else { + return self.ptr->size; + } +} + +static inline Length ts_subtree_total_size(Subtree self) { + return length_add(ts_subtree_padding(self), ts_subtree_size(self)); +} + +static inline uint32_t ts_subtree_total_bytes(Subtree self) { + return ts_subtree_total_size(self).bytes; +} + +static inline uint32_t ts_subtree_bytes_scanned(Subtree self) { + return self.data.is_inline + ? (uint32_t)self.data.padding_bytes + + (uint32_t)self.data.size_bytes + + (uint32_t)self.data.additional_bytes_scanned + : self.ptr->bytes_scanned; +} + +static inline uint32_t ts_subtree_child_count(Subtree self) { + return self.data.is_inline ? 0 : self.ptr->child_count; +} + +static inline uint32_t ts_subtree_node_count(Subtree self) { + return (self.data.is_inline || self.ptr->child_count == 0) ? 1 : self.ptr->node_count; +} + +static inline uint32_t ts_subtree_visible_child_count(Subtree self) { + if (ts_subtree_child_count(self) > 0) { + return self.ptr->visible_child_count; + } else { + return 0; + } +} + +static inline uint32_t ts_subtree_error_cost(Subtree self) { + if (ts_subtree_missing(self)) { + return ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY; + } else { + return self.data.is_inline ? 0 : self.ptr->error_cost; + } +} + +static inline int32_t ts_subtree_dynamic_precedence(Subtree self) { + return (self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->dynamic_precedence; +} + +static inline uint16_t ts_subtree_alias_sequence_id(Subtree self) { + if (ts_subtree_child_count(self) > 0) { + return self.ptr->alias_sequence_id; + } else { + return 0; + } +} + +static inline bool ts_subtree_fragile_left(Subtree self) { + return self.data.is_inline ? false : self.ptr->fragile_left; +} + +static inline bool ts_subtree_fragile_right(Subtree self) { + return self.data.is_inline ? false : self.ptr->fragile_right; +} + +static inline bool ts_subtree_has_external_tokens(Subtree self) { + return self.data.is_inline ? false : self.ptr->has_external_tokens; +} + +static inline bool ts_subtree_is_fragile(Subtree self) { + return self.data.is_inline ? false : (self.ptr->fragile_left || self.ptr->fragile_right); +} + +static inline bool ts_subtree_is_error(Subtree self) { + return ts_subtree_symbol(self) == ts_builtin_sym_error; +} + +static inline bool ts_subtree_is_eof(Subtree self) { + return ts_subtree_symbol(self) == ts_builtin_sym_end; +} + +static inline Subtree ts_subtree_from_mut(MutableSubtree self) { + Subtree result; + result.data = self.data; + return result; +} + +static inline MutableSubtree ts_subtree_to_mut_unsafe(Subtree self) { + MutableSubtree result; + result.data = self.data; + return result; } #ifdef __cplusplus diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 16b5a18b..0b1ccf62 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -7,7 +7,7 @@ static const unsigned PARENT_CACHE_CAPACITY = 32; -TSTree *ts_tree_new(const Subtree *root, const TSLanguage *language) { +TSTree *ts_tree_new(Subtree root, const TSLanguage *language) { TSTree *result = ts_malloc(sizeof(TSTree)); result->root = root; result->language = language; @@ -31,7 +31,7 @@ void ts_tree_delete(TSTree *self) { } TSNode ts_tree_root_node(const TSTree *self) { - return ts_node_new(self, self->root, self->root->padding, 0); + return ts_node_new(self, &self->root, ts_subtree_padding(self->root), 0); } const TSLanguage *ts_tree_language(const TSTree *self) { @@ -53,7 +53,7 @@ TSRange *ts_tree_get_changed_ranges(const TSTree *self, const TSTree *other, uin ts_tree_cursor_init(&cursor1, root); ts_tree_cursor_init(&cursor2, root); *count = ts_subtree_get_changed_ranges( - self->root, other->root, &cursor1, &cursor2, + &self->root, &other->root, &cursor1, &cursor2, self->language, &result ); array_delete(&cursor1.stack); @@ -85,7 +85,7 @@ void ts_tree_set_cached_parent(const TSTree *_self, const TSNode *node, const TS uint32_t index = (self->parent_cache_start + self->parent_cache_size) % PARENT_CACHE_CAPACITY; self->parent_cache[index] = (ParentCacheEntry) { .child = node->id, - .parent = parent->id, + .parent = (const Subtree *)parent->id, .position = { parent->context[0], {parent->context[1], parent->context[2]} diff --git a/src/runtime/tree.h b/src/runtime/tree.h index 99481d88..5a7e6547 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -13,14 +13,14 @@ typedef struct { } ParentCacheEntry; struct TSTree { - const Subtree *root; + Subtree root; const TSLanguage *language; ParentCacheEntry *parent_cache; uint32_t parent_cache_start; uint32_t parent_cache_size; }; -TSTree *ts_tree_new(const Subtree *root, const TSLanguage *language); +TSTree *ts_tree_new(Subtree root, const TSLanguage *language); TSNode ts_node_new(const TSTree *, const Subtree *, Length, TSSymbol); TSNode ts_tree_get_cached_parent(const TSTree *, const TSNode *); void ts_tree_set_cached_parent(const TSTree *, const TSNode *, const TSNode *); diff --git a/src/runtime/tree_cursor.c b/src/runtime/tree_cursor.c index 93a8be7f..3a7df9f4 100644 --- a/src/runtime/tree_cursor.c +++ b/src/runtime/tree_cursor.c @@ -5,7 +5,7 @@ #include "runtime/tree.h" typedef struct { - const Subtree *parent; + Subtree parent; const TSTree *tree; Length position; uint32_t child_index; @@ -17,13 +17,16 @@ typedef struct { static inline ChildIterator ts_tree_cursor_iterate_children(const TreeCursor *self) { TreeCursorEntry *last_entry = array_back(&self->stack); + if (ts_subtree_child_count(*last_entry->subtree) == 0) { + return (ChildIterator) {NULL_SUBTREE, self->tree, length_zero(), 0, 0, NULL}; + } const TSSymbol *alias_sequence = ts_language_alias_sequence( self->tree->language, - last_entry->subtree->alias_sequence_id + last_entry->subtree->ptr->alias_sequence_id ); return (ChildIterator) { .tree = self->tree, - .parent = last_entry->subtree, + .parent = *last_entry->subtree, .position = last_entry->position, .child_index = 0, .structural_child_index = 0, @@ -34,26 +37,27 @@ static inline ChildIterator ts_tree_cursor_iterate_children(const TreeCursor *se static inline bool ts_tree_cursor_child_iterator_next(ChildIterator *self, TreeCursorEntry *result, bool *visible) { - if (self->child_index == self->parent->child_count) return false; - const Subtree *child = self->parent->children[self->child_index]; + if (!self->parent.ptr || self->child_index == self->parent.ptr->child_count) return false; + const Subtree *child = &self->parent.ptr->children[self->child_index]; *result = (TreeCursorEntry) { .subtree = child, .position = self->position, .child_index = self->child_index, .structural_child_index = self->structural_child_index, }; - *visible = child->visible; - if (!child->extra && self->alias_sequence) { + *visible = ts_subtree_visible(*child); + bool extra = ts_subtree_extra(*child); + if (!extra && self->alias_sequence) { *visible |= self->alias_sequence[self->structural_child_index]; } - self->position = length_add(self->position, child->size); + self->position = length_add(self->position, ts_subtree_size(*child)); self->child_index++; - if (!child->extra) self->structural_child_index++; + if (!extra) self->structural_child_index++; - if (self->child_index < self->parent->child_count) { - const Subtree *child = self->parent->children[self->child_index]; - self->position = length_add(self->position, child->padding); + if (self->child_index < self->parent.ptr->child_count) { + Subtree next_child = self->parent.ptr->children[self->child_index]; + self->position = length_add(self->position, ts_subtree_padding(next_child)); } return true; @@ -104,7 +108,7 @@ bool ts_tree_cursor_goto_first_child(TSTreeCursor *_self) { return true; } - if (entry.subtree->child_count > 0 && entry.subtree->visible_child_count > 0) { + if (ts_subtree_visible_child_count(*entry.subtree) > 0) { array_push(&self->stack, entry); did_descend = true; break; @@ -128,11 +132,9 @@ int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *_self, uint32_t g TreeCursorEntry entry; ChildIterator iterator = ts_tree_cursor_iterate_children(self); while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) { - uint32_t end_byte = entry.position.bytes + entry.subtree->size.bytes; + uint32_t end_byte = entry.position.bytes + ts_subtree_size(*entry.subtree).bytes; bool at_goal = end_byte > goal_byte; - uint32_t visible_child_count = entry.subtree->child_count > 0 - ? entry.subtree->visible_child_count - : 0; + uint32_t visible_child_count = ts_subtree_visible_child_count(*entry.subtree); if (at_goal) { if (visible) { @@ -183,7 +185,7 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *_self) { return true; } - if (entry.subtree->child_count > 0 && entry.subtree->visible_child_count > 0) { + if (ts_subtree_visible_child_count(*entry.subtree)) { array_push(&self->stack, entry); ts_tree_cursor_goto_first_child(_self); return true; @@ -204,11 +206,11 @@ bool ts_tree_cursor_goto_parent(TSTreeCursor *_self) { TreeCursorEntry *parent_entry = &self->stack.contents[i - 1]; const TSSymbol *alias_sequence = ts_language_alias_sequence( self->tree->language, - parent_entry->subtree->alias_sequence_id + parent_entry->subtree->ptr->alias_sequence_id ); is_aliased = alias_sequence && alias_sequence[entry->structural_child_index]; } - if (entry->subtree->visible || is_aliased) { + if (ts_subtree_visible(*entry->subtree) || is_aliased) { self->stack.size = i + 1; return true; } @@ -224,9 +226,9 @@ TSNode ts_tree_cursor_current_node(const TSTreeCursor *_self) { TreeCursorEntry *parent_entry = &self->stack.contents[self->stack.size - 2]; const TSSymbol *alias_sequence = ts_language_alias_sequence( self->tree->language, - parent_entry->subtree->alias_sequence_id + parent_entry->subtree->ptr->alias_sequence_id ); - if (alias_sequence && !last_entry->subtree->extra) { + if (alias_sequence && !ts_subtree_extra(*last_entry->subtree)) { alias_symbol = alias_sequence[last_entry->structural_child_index]; } } diff --git a/test/helpers/point_helpers.cc b/test/helpers/point_helpers.cc index eabd0a82..ace303a0 100644 --- a/test/helpers/point_helpers.cc +++ b/test/helpers/point_helpers.cc @@ -35,6 +35,14 @@ bool operator>(const TSPoint &left, const TSPoint &right) { return right < left; } +Length operator*(const Length &length, uint32_t factor) { + return {length.bytes * factor, {0, length.extent.column * factor}}; +} + +Length operator+(const Length &left, const Length &right) { + return length_add(left, right); +} + std::ostream &operator<<(std::ostream &stream, const TSPoint &point) { return stream << "{" << point.row << ", " << point.column << "}"; } diff --git a/test/helpers/point_helpers.h b/test/helpers/point_helpers.h index 58558663..a64abc0d 100644 --- a/test/helpers/point_helpers.h +++ b/test/helpers/point_helpers.h @@ -14,6 +14,10 @@ bool operator==(const TSRange &left, const TSRange &right); bool operator==(const Length &left, const Length &right); +Length operator*(const Length &length, uint32_t factor); + +Length operator+(const Length &left, const Length &right); + std::ostream &operator<<(std::ostream &stream, const TSPoint &point); std::ostream &operator<<(std::ostream &stream, const TSRange &range); diff --git a/test/helpers/tree_helpers.cc b/test/helpers/tree_helpers.cc index 9d58b831..e8b3ace4 100644 --- a/test/helpers/tree_helpers.cc +++ b/test/helpers/tree_helpers.cc @@ -14,18 +14,18 @@ const char *symbol_names[24] = { "twenty-two", "twenty-three" }; -SubtreeArray *tree_array(std::vector trees) { +SubtreeArray *tree_array(std::vector trees) { static SubtreeArray result; result.capacity = trees.size(); result.size = trees.size(); - result.contents = (const Subtree **)calloc(trees.size(), sizeof(Subtree *)); + result.contents = (Subtree *)calloc(trees.size(), sizeof(Subtree)); for (size_t i = 0; i < trees.size(); i++) { result.contents[i] = trees[i]; } return &result; } -ostream &operator<<(std::ostream &stream, const Subtree *tree) { +ostream &operator<<(std::ostream &stream, Subtree tree) { static TSLanguage DUMMY_LANGUAGE = {}; DUMMY_LANGUAGE.symbol_names = symbol_names; char *string = ts_subtree_string(tree, &DUMMY_LANGUAGE, false); @@ -52,13 +52,10 @@ bool operator==(const TSNode &left, const TSNode &right) { ts_node_start_point(left) == ts_node_start_point(right); } -bool operator==(const std::vector &vec, const SubtreeArray &array) { - if (vec.size() != array.size) - return false; - for (size_t i = 0; i < array.size; i++) - if (array.contents[i] != vec[i]) - return false; - return true; +bool operator==(const std::vector &vec, const SubtreeArray &array) { + return + vec.size() == array.size && + std::memcmp(vec.data(), array.contents, array.size * sizeof(Subtree)) == 0; } void assert_consistent_tree_sizes(TSNode node) { diff --git a/test/helpers/tree_helpers.h b/test/helpers/tree_helpers.h index 00740faf..1e836eef 100644 --- a/test/helpers/tree_helpers.h +++ b/test/helpers/tree_helpers.h @@ -6,12 +6,12 @@ #include extern const char *symbol_names[24]; -SubtreeArray *tree_array(std::vector trees); +SubtreeArray *tree_array(std::vector trees); -std::ostream &operator<<(std::ostream &stream, const Subtree *tree); +std::ostream &operator<<(std::ostream &stream, Subtree tree); std::ostream &operator<<(std::ostream &stream, const TSNode &node); bool operator==(const TSNode &left, const TSNode &right); -bool operator==(const std::vector &right, const SubtreeArray &array); +bool operator==(const std::vector &right, const SubtreeArray &array); void assert_consistent_tree_sizes(TSNode node); diff --git a/test/runtime/parser_test.cc b/test/runtime/parser_test.cc index 4932f155..7c87bd02 100644 --- a/test/runtime/parser_test.cc +++ b/test/runtime/parser_test.cc @@ -839,7 +839,6 @@ describe("Parser", [&]() { AssertThat(ts_node_type(interpolation_node1), Equals("template_substitution")); AssertThat(ts_node_type(interpolation_node2), Equals("template_substitution")); AssertThat(ts_node_type(close_quote_node), Equals("`")); - ts_tree_delete(js_tree); TSRange included_ranges[] = { { @@ -863,6 +862,7 @@ describe("Parser", [&]() { }; ts_parser_set_included_ranges(parser, included_ranges, 3); + ts_tree_delete(js_tree); ts_parser_set_language(parser, load_real_language("html")); tree = ts_parser_parse_string(parser, nullptr, source_code.c_str(), source_code.size()); diff --git a/test/runtime/stack_test.cc b/test/runtime/stack_test.cc index 0330f318..aa710237 100644 --- a/test/runtime/stack_test.cc +++ b/test/runtime/stack_test.cc @@ -19,10 +19,6 @@ enum { symbol9, symbol10 }; -Length operator*(const Length &length, uint32_t factor) { - return {length.bytes * factor, {0, length.extent.column * factor}}; -} - void free_slice_array(SubtreePool *pool, StackSliceArray *slices) { for (size_t i = 0; i < slices->size; i++) { StackSlice slice = slices->contents[i]; @@ -44,8 +40,8 @@ void free_slice_array(SubtreePool *pool, StackSliceArray *slices) { } } -Subtree *mutate(const Subtree *subtree) { - return (Subtree *)subtree; +SubtreeHeapData *mutate(Subtree subtree) { + return ts_subtree_to_mut_unsafe(subtree).ptr; } struct StackEntry { @@ -73,7 +69,7 @@ START_TEST describe("Stack", [&]() { Stack *stack; const size_t subtree_count = 11; - const Subtree *subtrees[subtree_count]; + Subtree subtrees[subtree_count]; Length tree_len = {3, {0, 3}}; SubtreePool pool; @@ -92,7 +88,7 @@ describe("Stack", [&]() { &pool, i + 1, length_zero(), tree_len, 0, TS_TREE_STATE_NONE, true, false, &dummy_language ); - ts_external_scanner_state_init(&((Subtree *)subtrees[i])->external_scanner_state, nullptr, 0); + ts_external_scanner_state_init(&mutate(subtrees[i])->external_scanner_state, nullptr, 0); } }); @@ -107,7 +103,7 @@ describe("Stack", [&]() { AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); }); - auto push = [&](StackVersion version, const Subtree *tree, TSStateId state) { + auto push = [&](StackVersion version, Subtree tree, TSStateId state) { ts_subtree_retain(tree); ts_stack_push(stack, version, tree, false, state); }; @@ -269,7 +265,7 @@ describe("Stack", [&]() { StackSlice slice = pop.contents[0]; AssertThat(slice.version, Equals(1)); - AssertThat(slice.subtrees, Equals(vector({ subtrees[1], subtrees[2] }))); + AssertThat(slice.subtrees, Equals(vector({ subtrees[1], subtrees[2] }))); AssertThat(ts_stack_state(stack, 1), Equals(stateA)); free_slice_array(&pool,&pop); @@ -285,7 +281,7 @@ describe("Stack", [&]() { AssertThat(pop.size, Equals(1)); StackSlice slice = pop.contents[0]; - AssertThat(slice.subtrees, Equals(vector({ subtrees[0], subtrees[1], subtrees[2] }))); + AssertThat(slice.subtrees, Equals(vector({ subtrees[0], subtrees[1], subtrees[2] }))); AssertThat(ts_stack_state(stack, 1), Equals(1)); free_slice_array(&pool,&pop); @@ -330,11 +326,11 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.subtrees, Equals(vector({ subtrees[2], subtrees[3], subtrees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[2], subtrees[3], subtrees[10] }))); StackSlice slice2 = pop.contents[1]; AssertThat(slice2.version, Equals(2)); - AssertThat(slice2.subtrees, Equals(vector({ subtrees[5], subtrees[6], subtrees[10] }))); + AssertThat(slice2.subtrees, Equals(vector({ subtrees[5], subtrees[6], subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(3)); AssertThat(get_stack_entries(stack, 0), Equals(vector({ @@ -374,7 +370,7 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.subtrees, Equals(vector({ subtrees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(2)); AssertThat(ts_stack_state(stack, 0), Equals(stateI)); @@ -396,11 +392,11 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.subtrees, Equals(vector({ subtrees[1], subtrees[2], subtrees[3], subtrees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[1], subtrees[2], subtrees[3], subtrees[10] }))); StackSlice slice2 = pop.contents[1]; AssertThat(slice2.version, Equals(1)); - AssertThat(slice2.subtrees, Equals(vector({ subtrees[4], subtrees[5], subtrees[6], subtrees[10] }))); + AssertThat(slice2.subtrees, Equals(vector({ subtrees[4], subtrees[5], subtrees[6], subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(2)); AssertThat(ts_stack_state(stack, 0), Equals(stateI)); @@ -451,15 +447,15 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.subtrees, Equals(vector({ subtrees[3], subtrees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[3], subtrees[10] }))); StackSlice slice2 = pop.contents[1]; AssertThat(slice2.version, Equals(2)); - AssertThat(slice2.subtrees, Equals(vector({ subtrees[6], subtrees[10] }))); + AssertThat(slice2.subtrees, Equals(vector({ subtrees[6], subtrees[10] }))); StackSlice slice3 = pop.contents[2]; AssertThat(slice3.version, Equals(3)); - AssertThat(slice3.subtrees, Equals(vector({ subtrees[9], subtrees[10] }))); + AssertThat(slice3.subtrees, Equals(vector({ subtrees[9], subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(4)); AssertThat(ts_stack_state(stack, 0), Equals(stateI)); @@ -506,7 +502,7 @@ describe("Stack", [&]() { StackSliceArray pop = ts_stack_pop_pending(stack, 0); AssertThat(pop.size, Equals(1)); - AssertThat(pop.contents[0].subtrees, Equals(vector({ subtrees[1], subtrees[2], subtrees[3] }))); + AssertThat(pop.contents[0].subtrees, Equals(vector({ subtrees[1], subtrees[2], subtrees[3] }))); AssertThat(get_stack_entries(stack, 0), Equals(vector({ {stateA, 0}, @@ -541,16 +537,16 @@ describe("Stack", [&]() { }); it("allows the state to be retrieved", [&]() { - AssertThat(ts_stack_last_external_token(stack, 0), Equals(nullptr)); + AssertThat(ts_stack_last_external_token(stack, 0).ptr, Equals(nullptr)); ts_stack_set_last_external_token(stack, 0, subtrees[1]); - AssertThat(ts_stack_last_external_token(stack, 0), Equals(subtrees[1])); + AssertThat(ts_stack_last_external_token(stack, 0).ptr, Equals(subtrees[1].ptr)); ts_stack_copy_version(stack, 0); - AssertThat(ts_stack_last_external_token(stack, 1), Equals(subtrees[1])); + AssertThat(ts_stack_last_external_token(stack, 1).ptr, Equals(subtrees[1].ptr)); ts_stack_set_last_external_token(stack, 0, subtrees[2]); - AssertThat(ts_stack_last_external_token(stack, 0), Equals(subtrees[2])); + AssertThat(ts_stack_last_external_token(stack, 0).ptr, Equals(subtrees[2].ptr)); }); it("does not merge stack versions with different external token states", [&]() { diff --git a/test/runtime/subtree_test.cc b/test/runtime/subtree_test.cc index dd238f3e..3b5b145b 100644 --- a/test/runtime/subtree_test.cc +++ b/test/runtime/subtree_test.cc @@ -4,13 +4,13 @@ #include "runtime/subtree.h" #include "runtime/length.h" -void assert_consistent(const Subtree *tree) { - if (tree->child_count == 0) return; - AssertThat(tree->children[0]->padding, Equals(tree->padding)); +void assert_consistent(Subtree tree) { + if (ts_subtree_child_count(tree) == 0) return; + AssertThat(tree.ptr->children[0].ptr->padding, Equals(tree.ptr->padding)); Length total_children_size = length_zero(); - for (size_t i = 0; i < tree->child_count; i++) { - const Subtree *child = tree->children[i]; + for (size_t i = 0; i < tree.ptr->child_count; i++) { + Subtree child = tree.ptr->children[i]; assert_consistent(child); total_children_size = length_add(total_children_size, ts_subtree_total_size(child)); } @@ -54,124 +54,101 @@ describe("Subtree", []() { ); }; + auto new_node = [&](TSSymbol symbol, vector children) { + return ts_subtree_from_mut(ts_subtree_new_node( + &pool, symbol, tree_array(children), 0, &language + )); + }; + describe("new_node", [&]() { - const Subtree *tree1, *tree2, *parent1; + Subtree tree1, tree2; before_each([&]() { tree1 = new_leaf(symbol1, {2, {0, 1}}, {5, {0, 4}}, 0); tree2 = new_leaf(symbol2, {1, {0, 1}}, {3, {0, 3}}, 0); - - ts_subtree_retain(tree1); - ts_subtree_retain(tree2); - parent1 = ts_subtree_new_node(&pool, symbol3, tree_array({ - tree1, - tree2, - }), 0, &language); }); after_each([&]() { ts_subtree_release(&pool, tree1); ts_subtree_release(&pool, tree2); - ts_subtree_release(&pool, parent1); }); it("computes its size and padding based on its child nodes", [&]() { - AssertThat(parent1->size.bytes, Equals( - tree1->size.bytes + tree2->padding.bytes + tree2->size.bytes - )); - AssertThat(parent1->padding.bytes, Equals(tree1->padding.bytes)); + ts_subtree_retain(tree1); + ts_subtree_retain(tree2); + Subtree parent = new_node(symbol3, {tree1, tree2}); + + AssertThat( + ts_subtree_size(parent), + Equals( + ts_subtree_size(tree1) + ts_subtree_padding(tree2) + ts_subtree_size(tree2) + )); + AssertThat(ts_subtree_padding(parent), Equals(ts_subtree_padding(tree1))); + ts_subtree_release(&pool, parent); }); describe("when the first node is fragile on the left side", [&]() { - const Subtree *parent; - - before_each([&]() { - Subtree *mutable_tree1 = (Subtree *)tree1; - mutable_tree1->fragile_left = true; - mutable_tree1->extra = true; + it("records that it is fragile on the left side", [&]() { + MutableSubtree mutable_tree1 = ts_subtree_to_mut_unsafe(tree1); + mutable_tree1.ptr->fragile_left = true; + mutable_tree1.ptr->extra = true; ts_subtree_retain(tree1); ts_subtree_retain(tree2); - parent = ts_subtree_new_node(&pool, symbol3, tree_array({ - tree1, - tree2, - }), 0, &language); - }); + Subtree parent = new_node(symbol3, {tree1, tree2}); - after_each([&]() { + AssertThat(ts_subtree_fragile_left(parent), IsTrue()); + AssertThat(ts_subtree_fragile_right(parent), IsFalse()); ts_subtree_release(&pool, parent); }); - - it("records that it is fragile on the left side", [&]() { - AssertThat(parent->fragile_left, IsTrue()); - }); }); describe("when the last node is fragile on the right side", [&]() { - const Subtree *parent; - - before_each([&]() { - Subtree *mutable_tree2 = (Subtree *)tree2; - mutable_tree2->fragile_right = true; - mutable_tree2->extra = true; + it("records that it is fragile on the right side", [&]() { + MutableSubtree mutable_tree2 = ts_subtree_to_mut_unsafe(tree2); + mutable_tree2.ptr->fragile_right = true; + mutable_tree2.ptr->extra = true; ts_subtree_retain(tree1); ts_subtree_retain(tree2); - parent = ts_subtree_new_node(&pool, symbol3, tree_array({ - tree1, - tree2, - }), 0, &language); - }); + Subtree parent = new_node(symbol3, {tree1, tree2}); - after_each([&]() { + AssertThat(ts_subtree_fragile_left(parent), IsFalse()); + AssertThat(ts_subtree_fragile_right(parent), IsTrue()); ts_subtree_release(&pool, parent); }); - - it("records that it is fragile on the right side", [&]() { - AssertThat(parent->fragile_right, IsTrue()); - }); }); describe("when the outer nodes aren't fragile on their outer side", [&]() { - const Subtree *parent; - - before_each([&]() { - Subtree *mutable_tree1 = (Subtree *)tree1; - Subtree *mutable_tree2 = (Subtree *)tree2; - mutable_tree1->fragile_right = true; - mutable_tree2->fragile_left = true; + it("records that it is not fragile", [&]() { + MutableSubtree mutable_tree1 = ts_subtree_to_mut_unsafe(tree1); + MutableSubtree mutable_tree2 = ts_subtree_to_mut_unsafe(tree2); + mutable_tree1.ptr->fragile_right = true; + mutable_tree2.ptr->fragile_left = true; ts_subtree_retain(tree1); ts_subtree_retain(tree2); - parent = ts_subtree_new_node(&pool, symbol3, tree_array({ - tree1, - tree2, - }), 0, &language); - }); + Subtree parent = new_node(symbol3, {tree1, tree2}); - after_each([&]() { + AssertThat(ts_subtree_fragile_left(parent), IsFalse()); + AssertThat(ts_subtree_fragile_right(parent), IsFalse()); ts_subtree_release(&pool, parent); }); - - it("records that it is not fragile", [&]() { - AssertThat(parent->fragile_left, IsFalse()); - AssertThat(parent->fragile_right, IsFalse()); - }); }); }); describe("edit", [&]() { - const Subtree *tree; + Subtree tree; before_each([&]() { - tree = ts_subtree_new_node(&pool, symbol1, tree_array({ + tree = new_node(symbol1, { new_leaf(symbol2, {2, {0, 2}}, {3, {0, 3}}, 0), new_leaf(symbol3, {2, {0, 2}}, {3, {0, 3}}, 0), new_leaf(symbol4, {2, {0, 2}}, {3, {0, 3}}, 0), - }), 0, &language); + }); - AssertThat(tree->padding, Equals({2, {0, 2}})); - AssertThat(tree->size, Equals({13, {0, 13}})); + AssertThat(tree.ptr->padding, Equals({2, {0, 2}})); + AssertThat(tree.ptr->size, Equals({13, {0, 13}})); }); after_each([&]() { @@ -188,21 +165,21 @@ describe("Subtree", []() { edit.new_end_point = {0, 2}; ts_subtree_retain(tree); - const Subtree *new_tree = ts_subtree_edit(tree, &edit, &pool); + Subtree new_tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); assert_consistent(new_tree); - AssertThat(tree->has_changes, IsFalse()); - AssertThat(tree->padding, Equals({2, {0, 2}})); - AssertThat(tree->size, Equals({13, {0, 13}})); + AssertThat(ts_subtree_has_changes(tree), IsFalse()); + AssertThat(ts_subtree_padding(tree), Equals({2, {0, 2}})); + AssertThat(ts_subtree_size(tree), Equals({13, {0, 13}})); - AssertThat(tree->children[0]->has_changes, IsFalse()); - AssertThat(tree->children[0]->padding, Equals({2, {0, 2}})); - AssertThat(tree->children[0]->size, Equals({3, {0, 3}})); + AssertThat(ts_subtree_has_changes(tree.ptr->children[0]), IsFalse()); + AssertThat(ts_subtree_padding(tree.ptr->children[0]), Equals({2, {0, 2}})); + AssertThat(ts_subtree_size(tree.ptr->children[0]), Equals({3, {0, 3}})); - AssertThat(tree->children[1]->has_changes, IsFalse()); - AssertThat(tree->children[1]->padding, Equals({2, {0, 2}})); - AssertThat(tree->children[1]->size, Equals({3, {0, 3}})); + AssertThat(ts_subtree_has_changes(tree.ptr->children[1]), IsFalse()); + AssertThat(ts_subtree_padding(tree.ptr->children[1]), Equals({2, {0, 2}})); + AssertThat(ts_subtree_size(tree.ptr->children[1]), Equals({3, {0, 3}})); ts_subtree_release(&pool, new_tree); }); @@ -220,17 +197,17 @@ describe("Subtree", []() { tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); - AssertThat(tree->has_changes, IsTrue()); - AssertThat(tree->padding, Equals({3, {0, 3}})); - AssertThat(tree->size, Equals({13, {0, 13}})); + AssertThat(ts_subtree_has_changes(tree), IsTrue()); + AssertThat(ts_subtree_padding(tree), Equals({3, {0, 3}})); + AssertThat(ts_subtree_size(tree), Equals({13, {0, 13}})); - AssertThat(tree->children[0]->has_changes, IsTrue()); - AssertThat(tree->children[0]->padding, Equals({3, {0, 3}})); - AssertThat(tree->children[0]->size, Equals({3, {0, 3}})); + AssertThat(ts_subtree_has_changes(tree.ptr->children[0]), IsTrue()); + AssertThat(ts_subtree_padding(tree.ptr->children[0]), Equals({3, {0, 3}})); + AssertThat(ts_subtree_size(tree.ptr->children[0]), Equals({3, {0, 3}})); - AssertThat(tree->children[1]->has_changes, IsFalse()); - AssertThat(tree->children[1]->padding, Equals({2, {0, 2}})); - AssertThat(tree->children[1]->size, Equals({3, {0, 3}})); + AssertThat(ts_subtree_has_changes(tree.ptr->children[1]), IsFalse()); + AssertThat(ts_subtree_padding(tree.ptr->children[1]), Equals({2, {0, 2}})); + AssertThat(ts_subtree_size(tree.ptr->children[1]), Equals({3, {0, 3}})); }); }); @@ -247,13 +224,13 @@ describe("Subtree", []() { tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); - AssertThat(tree->has_changes, IsTrue()); - AssertThat(tree->padding, Equals({5, {0, 5}})); - AssertThat(tree->size, Equals({11, {0, 11}})); + AssertThat(ts_subtree_has_changes(tree), IsTrue()); + AssertThat(ts_subtree_padding(tree), Equals({5, {0, 5}})); + AssertThat(ts_subtree_size(tree), Equals({11, {0, 11}})); - AssertThat(tree->children[0]->has_changes, IsTrue()); - AssertThat(tree->children[0]->padding, Equals({5, {0, 5}})); - AssertThat(tree->children[0]->size, Equals({1, {0, 1}})); + AssertThat(ts_subtree_has_changes(tree.ptr->children[0]), IsTrue()); + AssertThat(ts_subtree_padding(tree.ptr->children[0]), Equals({5, {0, 5}})); + AssertThat(ts_subtree_size(tree.ptr->children[0]), Equals({1, {0, 1}})); }); }); @@ -270,15 +247,15 @@ describe("Subtree", []() { tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); - AssertThat(tree->has_changes, IsTrue()); - AssertThat(tree->padding, Equals({4, {0, 4}})); - AssertThat(tree->size, Equals({13, {0, 13}})); + AssertThat(ts_subtree_has_changes(tree), IsTrue()); + AssertThat(ts_subtree_padding(tree), Equals({4, {0, 4}})); + AssertThat(ts_subtree_size(tree), Equals({13, {0, 13}})); - AssertThat(tree->children[0]->has_changes, IsTrue()); - AssertThat(tree->children[0]->padding, Equals({4, {0, 4}})); - AssertThat(tree->children[0]->size, Equals({3, {0, 3}})); + AssertThat(ts_subtree_has_changes(tree.ptr->children[0]), IsTrue()); + AssertThat(ts_subtree_padding(tree.ptr->children[0]), Equals({4, {0, 4}})); + AssertThat(ts_subtree_size(tree.ptr->children[0]), Equals({3, {0, 3}})); - AssertThat(tree->children[1]->has_changes, IsFalse()); + AssertThat(ts_subtree_has_changes(tree.ptr->children[1]), IsFalse()); }); }); @@ -295,15 +272,15 @@ describe("Subtree", []() { tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); - AssertThat(tree->has_changes, IsTrue()); - AssertThat(tree->padding, Equals({2, {0, 2}})); - AssertThat(tree->size, Equals({16, {0, 16}})); + AssertThat(ts_subtree_has_changes(tree), IsTrue()); + AssertThat(ts_subtree_padding(tree), Equals({2, {0, 2}})); + AssertThat(ts_subtree_size(tree), Equals({16, {0, 16}})); - AssertThat(tree->children[0]->has_changes, IsTrue()); - AssertThat(tree->children[0]->padding, Equals({2, {0, 2}})); - AssertThat(tree->children[0]->size, Equals({6, {0, 6}})); + AssertThat(ts_subtree_has_changes(tree.ptr->children[0]), IsTrue()); + AssertThat(ts_subtree_padding(tree.ptr->children[0]), Equals({2, {0, 2}})); + AssertThat(ts_subtree_size(tree.ptr->children[0]), Equals({6, {0, 6}})); - AssertThat(tree->children[1]->has_changes, IsFalse()); + AssertThat(ts_subtree_has_changes(tree.ptr->children[1]), IsFalse()); }); }); @@ -320,28 +297,28 @@ describe("Subtree", []() { tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); - AssertThat(tree->has_changes, IsTrue()); - AssertThat(tree->padding, Equals({4, {0, 4}})); - AssertThat(tree->size, Equals({4, {0, 4}})); + AssertThat(ts_subtree_has_changes(tree), IsTrue()); + AssertThat(ts_subtree_padding(tree), Equals({4, {0, 4}})); + AssertThat(ts_subtree_size(tree), Equals({4, {0, 4}})); - AssertThat(tree->children[0]->has_changes, IsTrue()); - AssertThat(tree->children[0]->padding, Equals({4, {0, 4}})); - AssertThat(tree->children[0]->size, Equals({0, {0, 0}})); + AssertThat(ts_subtree_has_changes(tree.ptr->children[0]), IsTrue()); + AssertThat(ts_subtree_padding(tree.ptr->children[0]), Equals({4, {0, 4}})); + AssertThat(ts_subtree_size(tree.ptr->children[0]), Equals({0, {0, 0}})); - AssertThat(tree->children[1]->has_changes, IsTrue()); - AssertThat(tree->children[1]->padding, Equals({0, {0, 0}})); - AssertThat(tree->children[1]->size, Equals({0, {0, 0}})); + AssertThat(ts_subtree_has_changes(tree.ptr->children[1]), IsTrue()); + AssertThat(ts_subtree_padding(tree.ptr->children[1]), Equals({0, {0, 0}})); + AssertThat(ts_subtree_size(tree.ptr->children[1]), Equals({0, {0, 0}})); - AssertThat(tree->children[2]->has_changes, IsTrue()); - AssertThat(tree->children[2]->padding, Equals({1, {0, 1}})); - AssertThat(tree->children[2]->size, Equals({3, {0, 3}})); + AssertThat(ts_subtree_has_changes(tree.ptr->children[2]), IsTrue()); + AssertThat(ts_subtree_padding(tree.ptr->children[2]), Equals({1, {0, 1}})); + AssertThat(ts_subtree_size(tree.ptr->children[2]), Equals({3, {0, 3}})); }); }); describe("edits within a tree's range of scanned bytes", [&]() { it("marks preceding trees as changed", [&]() { - Subtree *mutable_child = (Subtree *)tree->children[0]; - mutable_child->bytes_scanned = 7; + MutableSubtree mutable_child = ts_subtree_to_mut_unsafe(tree.ptr->children[0]); + mutable_child.ptr->bytes_scanned = 7; TSInputEdit edit; edit.start_byte = 6; @@ -354,7 +331,7 @@ describe("Subtree", []() { tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); - AssertThat(tree->children[0]->has_changes, IsTrue()); + AssertThat(ts_subtree_has_changes(tree.ptr->children[0]), IsTrue()); }); }); @@ -371,9 +348,9 @@ describe("Subtree", []() { tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); - AssertThat(tree->size.bytes, Equals(14u)); - AssertThat(tree->children[2]->has_changes, IsTrue()); - AssertThat(tree->children[2]->size.bytes, Equals(4u)); + AssertThat(ts_subtree_size(tree).bytes, Equals(14u)); + AssertThat(ts_subtree_has_changes(tree.ptr->children[2]), IsTrue()); + AssertThat(ts_subtree_size(tree.ptr->children[2]).bytes, Equals(4u)); }); }); @@ -390,14 +367,14 @@ describe("Subtree", []() { tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); - AssertThat(tree->size.bytes, Equals(13u)); - AssertThat(tree->children[2]->size.bytes, Equals(3u)); + AssertThat(ts_subtree_size(tree).bytes, Equals(13u)); + AssertThat(ts_subtree_size(tree.ptr->children[2]).bytes, Equals(3u)); }); }); }); describe("eq", [&]() { - const Subtree *leaf; + Subtree leaf; before_each([&]() { leaf = new_leaf(symbol1, {2, {1, 1}}, {5, {1, 4}}, 0); @@ -408,20 +385,14 @@ describe("Subtree", []() { }); it("returns true for identical trees", [&]() { - const Subtree *leaf_copy = new_leaf(symbol1, {2, {1, 1}}, {5, {1, 4}}, 0); + Subtree leaf_copy = new_leaf(symbol1, {2, {1, 1}}, {5, {1, 4}}, 0); AssertThat(ts_subtree_eq(leaf, leaf_copy), IsTrue()); - const Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({ - leaf, - leaf_copy, - }), 0, &language); + Subtree parent = new_node(symbol2, {leaf, leaf_copy}); ts_subtree_retain(leaf); ts_subtree_retain(leaf_copy); - const Subtree *parent_copy = ts_subtree_new_node(&pool, symbol2, tree_array({ - leaf, - leaf_copy, - }), 0, &language); + Subtree parent_copy = new_node(symbol2, {leaf, leaf_copy}); ts_subtree_retain(leaf); ts_subtree_retain(leaf_copy); @@ -433,11 +404,11 @@ describe("Subtree", []() { }); it("returns false for trees with different symbols", [&]() { - const Subtree *different_leaf = new_leaf( - leaf->symbol + 1, - leaf->padding, - leaf->size, - leaf->bytes_scanned + Subtree different_leaf = new_leaf( + ts_subtree_symbol(leaf) + 1, + ts_subtree_padding(leaf), + ts_subtree_size(leaf), + ts_subtree_bytes_scanned(leaf) ); AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); @@ -445,40 +416,39 @@ describe("Subtree", []() { }); it("returns false for trees with different options", [&]() { - const Subtree *different_leaf = new_leaf( - leaf->symbol, leaf->padding, leaf->size, leaf->bytes_scanned + Subtree different_leaf = new_leaf( + ts_subtree_symbol(leaf), + ts_subtree_padding(leaf), + ts_subtree_size(leaf), + ts_subtree_bytes_scanned(leaf) ); - ((Subtree *)different_leaf)->visible = !leaf->visible; + ts_subtree_to_mut_unsafe(different_leaf).ptr->visible = !ts_subtree_visible(leaf); AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); ts_subtree_release(&pool, different_leaf); }); it("returns false for trees with different paddings or sizes", [&]() { - const Subtree *different_leaf = new_leaf( - leaf->symbol, {}, leaf->size, leaf->bytes_scanned + Subtree different_leaf = new_leaf( + ts_subtree_symbol(leaf), + {}, + ts_subtree_size(leaf), + ts_subtree_bytes_scanned(leaf) ); AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); ts_subtree_release(&pool, different_leaf); - different_leaf = new_leaf(symbol1, leaf->padding, {}, leaf->bytes_scanned); + different_leaf = new_leaf(symbol1, ts_subtree_padding(leaf), {}, ts_subtree_bytes_scanned(leaf)); AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); ts_subtree_release(&pool, different_leaf); }); it("returns false for trees with different children", [&]() { - const Subtree *leaf2 = new_leaf(symbol2, {1, {0, 1}}, {3, {0, 3}}, 0); - - const Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({ - leaf, - leaf2, - }), 0, &language); + Subtree leaf2 = new_leaf(symbol2, {1, {0, 1}}, {3, {0, 3}}, 0); + Subtree parent = new_node(symbol2, {leaf, leaf2}); ts_subtree_retain(leaf); ts_subtree_retain(leaf2); - const Subtree *different_parent = ts_subtree_new_node(&pool, symbol2, tree_array({ - leaf2, - leaf, - }), 0, &language); + Subtree different_parent = new_node(symbol2, {leaf2, leaf}); ts_subtree_retain(leaf2); ts_subtree_retain(leaf); @@ -495,31 +465,33 @@ describe("Subtree", []() { Length padding = {1, {0, 1}}; Length size = {2, {0, 2}}; - auto make_external = [](const Subtree *_tree) { - Subtree *tree = (Subtree *)_tree; - ts_external_scanner_state_init(&tree->external_scanner_state, NULL, 0); + auto make_external = [](Subtree tree) { + ts_external_scanner_state_init( + &ts_subtree_to_mut_unsafe(tree).ptr->external_scanner_state, + NULL, 0 + ); return tree; }; it("returns the last serialized external token state in the given tree", [&]() { - const Subtree *tree1, *tree2, *tree3, *tree4, *tree5, *tree6, *tree7, *tree8, *tree9; + Subtree tree1, tree2, tree3, tree4, tree5, tree6, tree7, tree8, tree9; - tree1 = ts_subtree_new_node(&pool, symbol1, tree_array({ - (tree2 = ts_subtree_new_node(&pool, symbol2, tree_array({ + tree1 = new_node(symbol1, { + (tree2 = new_node(symbol2, { (tree3 = make_external(ts_subtree_new_leaf(&pool, symbol3, padding, size, 0, 0, true, false, &language))), (tree4 = new_leaf(symbol4, padding, size, 0)), (tree5 = new_leaf(symbol5, padding, size, 0)), - }), 0, &language)), - (tree6 = ts_subtree_new_node(&pool, symbol6, tree_array({ - (tree7 = ts_subtree_new_node(&pool, symbol7, tree_array({ + })), + (tree6 = new_node(symbol6, { + (tree7 = new_node(symbol7, { (tree8 = new_leaf(symbol8, padding, size, 0)), - }), 0, &language)), + })), (tree9 = new_leaf(symbol9, padding, size, 0)), - }), 0, &language)), - }), 0, &language); + })), + }); auto token = ts_subtree_last_external_token(tree1); - AssertThat(token, Equals(tree3)); + AssertThat(token.ptr, Equals(tree3.ptr)); ts_subtree_release(&pool, tree1); }); diff --git a/test/test_helper.h b/test/test_helper.h index 0cb2d41c..99db971a 100644 --- a/test/test_helper.h +++ b/test/test_helper.h @@ -16,6 +16,6 @@ using namespace tree_sitter; #define START_TEST go_bandit([]() { #define END_TEST }); -#define TREE_SITTER_WRAP_MALLOC +#define TREE_SITTER_TEST #endif // TEST_HELPER_