From f6208435915dc776f79809cc4ab5f538e3dcfa32 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 8 May 2018 11:42:15 -0700 Subject: [PATCH 01/31] Remove unused ReusableNode function --- src/runtime/parser.c | 10 ++++------ src/runtime/reusable_node.h | 11 +++++------ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 265354a2..ff194936 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -396,8 +396,7 @@ static void parser__set_cached_token(Parser *self, size_t byte_index, Tree *last } static bool parser__can_reuse_first_leaf(Parser *self, TSStateId state, Tree *tree, - TableEntry *table_entry, - ReusableNode *next_reusable_node) { + TableEntry *table_entry) { TSLexMode current_lex_mode = self->language->lex_modes[state]; // If the token was created in a state with the same set of lookaheads, it is reusable. @@ -462,14 +461,13 @@ static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId } ts_language_table_entry(self->language, *state, result->first_leaf.symbol, table_entry); - ReusableNode next_reusable_node = reusable_node_after_leaf(reusable_node); - if (!parser__can_reuse_first_leaf(self, *state, result, table_entry, &next_reusable_node)) { + if (!parser__can_reuse_first_leaf(self, *state, result, table_entry)) { LOG( "cant_reuse_node symbol:%s, first_leaf_symbol:%s", SYM_NAME(result->symbol), SYM_NAME(result->first_leaf.symbol) ); - *reusable_node = next_reusable_node; + reusable_node_pop_leaf(reusable_node); break; } @@ -480,7 +478,7 @@ static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId if ((result = parser__get_cached_token(self, position.bytes, last_external_token))) { ts_language_table_entry(self->language, *state, result->first_leaf.symbol, table_entry); - if (parser__can_reuse_first_leaf(self, *state, result, table_entry, NULL)) { + if (parser__can_reuse_first_leaf(self, *state, result, table_entry)) { ts_tree_retain(result); return result; } diff --git a/src/runtime/reusable_node.h b/src/runtime/reusable_node.h index 04b9af7e..4a6290f4 100644 --- a/src/runtime/reusable_node.h +++ b/src/runtime/reusable_node.h @@ -28,12 +28,11 @@ static inline void reusable_node_pop(ReusableNode *self) { } } -static inline ReusableNode reusable_node_after_leaf(const ReusableNode *self) { - ReusableNode result = *self; - while (result.tree->children.size > 0) - result.tree = result.tree->children.contents[0]; - reusable_node_pop(&result); - return result; +static inline void reusable_node_pop_leaf(ReusableNode *self) { + while (self->tree->children.size > 0) { + self->tree = self->tree->children.contents[0]; + } + reusable_node_pop(self); } static inline bool reusable_node_breakdown(ReusableNode *self) { From 8300f24fec6cd8e1a09b7ee21cac8cce5a29b25e Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 20 Apr 2018 10:00:00 -0700 Subject: [PATCH 02/31] Avoid slow test setup if seed flag is set to -1 --- test/integration/real_grammars.cc | 2 ++ test/integration/test_grammars.cc | 2 ++ test/test_helper.h | 2 ++ test/tests.cc | 11 ++++++----- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/test/integration/real_grammars.cc b/test/integration/real_grammars.cc index 37465add..e842ee0c 100644 --- a/test/integration/real_grammars.cc +++ b/test/integration/real_grammars.cc @@ -20,6 +20,8 @@ static void assert_correct_tree_size(TSDocument *document, string content) { START_TEST +if (TREE_SITTER_SEED == -1) return; + vector test_languages({ "javascript", "json", diff --git a/test/integration/test_grammars.cc b/test/integration/test_grammars.cc index 9dbefcd8..f4ba6d92 100644 --- a/test/integration/test_grammars.cc +++ b/test/integration/test_grammars.cc @@ -9,6 +9,8 @@ START_TEST +if (TREE_SITTER_SEED == -1) return; + string grammars_dir_path = join_path({"test", "fixtures", "test_grammars"}); vector test_languages = list_directory(grammars_dir_path); diff --git a/test/test_helper.h b/test/test_helper.h index 6947452b..0cb2d41c 100644 --- a/test/test_helper.h +++ b/test/test_helper.h @@ -5,6 +5,8 @@ #include "tree_sitter/compiler.h" #include "tree_sitter/runtime.h" +extern int TREE_SITTER_SEED; + namespace tree_sitter {} using namespace std; diff --git a/test/tests.cc b/test/tests.cc index cb9d6595..303f9059 100644 --- a/test/tests.cc +++ b/test/tests.cc @@ -1,17 +1,18 @@ #include "test_helper.h" #include "helpers/random_helpers.h" +int TREE_SITTER_SEED = 0; + int main(int argc, char *argv[]) { - int seed; const char *seed_env = getenv("TREE_SITTER_SEED"); if (seed_env) { - seed = atoi(seed_env); + TREE_SITTER_SEED = atoi(seed_env); } else { - seed = get_time_as_seed(); + TREE_SITTER_SEED = get_time_as_seed(); } - printf("Random seed: %d\n", seed); - random_reseed(seed); + printf("Random seed: %d\n", TREE_SITTER_SEED); + random_reseed(TREE_SITTER_SEED); return bandit::run(argc, argv); } From 973e4a44f0ade679598bc7499be2d2820bcfd355 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 10:16:10 -0700 Subject: [PATCH 03/31] Start work on removing parent pointers Co-Authored-By: Rick Winfrey --- include/tree_sitter/runtime.h | 8 +- src/runtime/array.h | 9 + src/runtime/document.c | 8 +- src/runtime/node.c | 307 +++++++++++------- src/runtime/node.h | 8 - src/runtime/parser.c | 54 +-- src/runtime/reusable_node.h | 95 ++++-- src/runtime/tree.c | 138 ++++---- src/runtime/tree.h | 14 +- .../prepare_grammar/extract_tokens_test.cc | 22 +- test/helpers/tree_helpers.cc | 25 +- test/runtime/document_test.cc | 2 +- test/runtime/node_test.cc | 40 +-- test/runtime/tree_test.cc | 4 +- 14 files changed, 410 insertions(+), 324 deletions(-) delete mode 100644 src/runtime/node.h diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index c3bef506..c4bc8ab5 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -63,8 +63,11 @@ typedef struct { } TSRange; typedef struct { - const void *data; - uint32_t offset[2]; + const void *subtree; + const TSDocument *document; + TSPoint position; + uint32_t byte; + TSSymbol alias_symbol; } TSNode; uint32_t ts_node_start_byte(TSNode); @@ -84,7 +87,6 @@ TSNode ts_node_child(TSNode, uint32_t); TSNode ts_node_named_child(TSNode, uint32_t); uint32_t ts_node_child_count(TSNode); uint32_t ts_node_named_child_count(TSNode); -uint32_t ts_node_child_index(TSNode); TSNode ts_node_next_sibling(TSNode); TSNode ts_node_next_named_sibling(TSNode); TSNode ts_node_prev_sibling(TSNode); diff --git a/src/runtime/array.h b/src/runtime/array.h index 4d2f478b..45b3adaa 100644 --- a/src/runtime/array.h +++ b/src/runtime/array.h @@ -58,6 +58,9 @@ extern "C" { #define array_pop(self) ((self)->contents[--(self)->size]) +#define array_assign(self, other) \ + array__assign((VoidArray *)(self), (const VoidArray *)(other), array__elem_size(self)) + // Private typedef Array(void) VoidArray; @@ -91,6 +94,12 @@ static inline void array__reserve(VoidArray *self, size_t element_size, uint32_t } } +static inline void array__assign(VoidArray *self, const VoidArray *other, size_t element_size) { + array__reserve(self, element_size, other->size); + self->size = other->size; + memcpy(self->contents, other->contents, self->size * element_size); +} + static inline void array__grow(VoidArray *self, size_t element_size) { if (self->size == self->capacity) { size_t new_capacity = self->capacity * 2; diff --git a/src/runtime/document.c b/src/runtime/document.c index d611d989..ddd89b43 100644 --- a/src/runtime/document.c +++ b/src/runtime/document.c @@ -1,5 +1,4 @@ #include "runtime/alloc.h" -#include "runtime/node.h" #include "runtime/tree.h" #include "runtime/parser.h" #include "runtime/string_input.h" @@ -171,7 +170,12 @@ void ts_document_invalidate(TSDocument *self) { } TSNode ts_document_root_node(const TSDocument *self) { - return ts_node_make(self->tree, 0, 0); + return (TSNode) { + .subtree = self->tree, + .document = self, + .position = {0, 0}, + .byte = 0, + }; } uint32_t ts_document_parse_count(const TSDocument *self) { diff --git a/src/runtime/node.c b/src/runtime/node.c index c825a104..246cd290 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -1,43 +1,91 @@ #include -#include "runtime/node.h" #include "runtime/tree.h" #include "runtime/document.h" +#include "runtime/language.h" -TSNode ts_node_make(const Tree *tree, uint32_t byte, uint32_t row) { - return (TSNode){.data = tree, .offset = { byte, row } }; -} +// NodeChildIterator -/* - * Private - */ +typedef struct { + const Tree *parent; + const TSDocument *document; + Length position; + uint32_t child_index; + uint32_t structural_child_index; + const TSSymbol *alias_sequence; +} NodeChildIterator; + +// TSNode - Private static inline TSNode ts_node__null() { - return ts_node_make(NULL, 0, 0); + return (TSNode) { + .subtree = NULL, + .document = NULL, + .position = {0, 0}, + .byte = 0, + }; } static inline const Tree *ts_node__tree(TSNode self) { - return self.data; + return self.subtree; } -static inline uint32_t ts_node__offset_byte(TSNode self) { - return self.offset[0]; +static inline NodeChildIterator ts_node_child_iterator_begin(const TSNode *node) { + const Tree *tree = ts_node__tree(*node); + const TSSymbol *alias_sequence = ts_language_alias_sequence( + node->document->parser.language, + tree->alias_sequence_id + ); + return (NodeChildIterator) { + .parent = tree, + .document = node->document, + .position = {node->byte, node->position}, + .child_index = 0, + .structural_child_index = 0, + .alias_sequence = alias_sequence, + }; } -static inline uint32_t ts_node__offset_row(TSNode self) { - return self.offset[1]; +static inline bool ts_node_child_iterator_next(NodeChildIterator *self, TSNode *result) { + if (self->child_index == self->parent->children.size) return false; + Tree *child = self->parent->children.contents[self->child_index]; + TSSymbol alias_symbol = 0; + if (!child->extra) { + if (self->alias_sequence) { + alias_symbol = self->alias_sequence[self->structural_child_index]; + } + self->structural_child_index++; + } + *result = (TSNode) { + .subtree = child, + .document = self->document, + .position = self->position.extent, + .byte = self->position.bytes, + .alias_symbol = alias_symbol, + }; + self->position = length_add(self->position, ts_tree_total_size(child)); + self->child_index++; + return true; } static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { const Tree *tree = ts_node__tree(self); if (include_anonymous) { - return tree->context.alias_symbol || tree->visible; + return tree->visible || self.alias_symbol; } else { - return tree->context.alias_is_named || (tree->visible && tree->named); + return ( + (tree->visible && tree->named) || + ( + self.alias_symbol && + ts_language_symbol_metadata( + self.document->parser.language, + self.alias_symbol + ).named + ) + ); } } -static inline uint32_t ts_node__relevant_child_count(TSNode self, - bool include_anonymous) { +static inline uint32_t ts_node__relevant_child_count(TSNode self, bool include_anonymous) { const Tree *tree = ts_node__tree(self); if (tree->children.size > 0) { if (include_anonymous) { @@ -50,44 +98,23 @@ static inline uint32_t ts_node__relevant_child_count(TSNode self, } } -static inline TSNode ts_node__direct_parent(TSNode self, uint32_t *index) { - const Tree *tree = ts_node__tree(self); - *index = tree->context.index; - return ts_node_make( - tree->context.parent, - ts_node__offset_byte(self) - tree->context.offset.bytes, - ts_node__offset_row(self) - tree->context.offset.extent.row - ); -} - -static inline TSNode ts_node__direct_child(TSNode self, uint32_t i) { - const Tree *child_tree = ts_node__tree(self)->children.contents[i]; - return ts_node_make( - child_tree, - ts_node__offset_byte(self) + child_tree->context.offset.bytes, - ts_node__offset_row(self) + child_tree->context.offset.extent.row - ); -} - -static inline TSNode ts_node__child(TSNode self, uint32_t child_index, - bool include_anonymous) { +static inline TSNode ts_node__child(TSNode self, uint32_t child_index, bool include_anonymous) { TSNode result = self; bool did_descend = true; while (did_descend) { did_descend = false; + TSNode child; uint32_t index = 0; - for (uint32_t i = 0; i < ts_node__tree(result)->children.size; i++) { - TSNode child = ts_node__direct_child(result, i); + NodeChildIterator iterator = ts_node_child_iterator_begin(&result); + while (ts_node_child_iterator_next(&iterator, &child)) { if (ts_node__is_relevant(child, include_anonymous)) { - if (index == child_index) - return child; + if (index == child_index) return child; index++; } else { uint32_t grandchild_index = child_index - index; - uint32_t grandchild_count = - ts_node__relevant_child_count(child, include_anonymous); + uint32_t grandchild_count = ts_node__relevant_child_count(child, include_anonymous); if (grandchild_index < grandchild_count) { did_descend = true; result = child; @@ -102,48 +129,80 @@ static inline TSNode ts_node__child(TSNode self, uint32_t child_index, return ts_node__null(); } -static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) { - TSNode result = self; - - do { - uint32_t index; - result = ts_node__direct_parent(result, &index); - if (!result.data) +static inline bool ts_node__last_child_before(TSNode self, TSNode target, + bool include_anonymous, TSNode *result) { + TSNode child; + TSNode earlier_child = ts_node__null(); + bool earlier_child_is_relevant = false; + bool found_child_containing_target = false; + NodeChildIterator iterator = ts_node_child_iterator_begin(&self); + uint32_t target_end_byte = ts_node_end_byte(target); + while (ts_node_child_iterator_next(&iterator, &child)) { + if (iterator.position.bytes >= target_end_byte) { + found_child_containing_target = true; break; - - for (uint32_t i = index - 1; i + 1 > 0; i--) { - TSNode child = ts_node__direct_child(result, i); - if (ts_node__is_relevant(child, include_anonymous)) - return child; - uint32_t grandchild_count = - ts_node__relevant_child_count(child, include_anonymous); - if (grandchild_count > 0) - return ts_node__child(child, grandchild_count - 1, include_anonymous); } - } while (!ts_node__tree(result)->visible); - return ts_node__null(); + if (ts_node__is_relevant(child, include_anonymous)) { + earlier_child = child; + earlier_child_is_relevant = true; + } else if (ts_node__relevant_child_count(child, include_anonymous) > 0) { + earlier_child = child; + earlier_child_is_relevant = false; + } + } + + if (found_child_containing_target && child.subtree != target.subtree) { + if (ts_node__last_child_before(child, target, include_anonymous, result)) { + return true; + } + } + + if (earlier_child_is_relevant) { + *result = earlier_child; + return true; + } + + if (earlier_child.subtree) { + return ts_node__last_child_before(earlier_child, target, include_anonymous, result); + } + + return false; +} + +static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) { + TSNode result = ts_node__null(); + TSNode parent = ts_node_parent(self); + if (parent.subtree) { + ts_node__last_child_before(parent, self, include_anonymous, &result); + } + return result; } static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) { - TSNode result = self; + TSNode node = ts_node_parent(self); + if (!node.subtree) return ts_node__null(); + uint32_t end_byte = ts_node_end_byte(self); - do { - uint32_t index; - result = ts_node__direct_parent(result, &index); - if (!result.data) - break; + bool did_descend = true; + while (did_descend) { + did_descend = false; - for (uint32_t i = index + 1; i < ts_node__tree(result)->children.size; i++) { - TSNode child = ts_node__direct_child(result, i); - if (ts_node__is_relevant(child, include_anonymous)) - return child; - uint32_t grandchild_count = - ts_node__relevant_child_count(child, include_anonymous); - if (grandchild_count > 0) - return ts_node__child(child, 0, include_anonymous); + TSNode child; + NodeChildIterator iterator = ts_node_child_iterator_begin(&node); + while (ts_node_child_iterator_next(&iterator, &child)) { + if (iterator.position.bytes > end_byte && child.subtree != self.subtree) { + if (ts_node__is_relevant(child, include_anonymous)) { + return child; + } + if (ts_node__relevant_child_count(child, include_anonymous) > 0) { + node = child; + did_descend = true; + break; + } + } } - } while (!ts_node__tree(result)->visible); + } return ts_node__null(); } @@ -160,8 +219,9 @@ static inline TSNode ts_node__first_child_for_byte(TSNode self, uint32_t goal, while (did_descend) { did_descend = false; - for (uint32_t i = 0; i < ts_node__tree(node)->children.size; i++) { - TSNode child = ts_node__direct_child(node, i); + TSNode child; + NodeChildIterator iterator = ts_node_child_iterator_begin(&node); + while (ts_node_child_iterator_next(&iterator, &child)) { if (ts_node_end_byte(child) > goal) { if (ts_node__is_relevant(child, include_anonymous)) { return child; @@ -187,10 +247,11 @@ static inline TSNode ts_node__descendant_for_byte_range(TSNode self, uint32_t mi while (did_descend) { did_descend = false; - for (uint32_t i = 0, n = ts_node__tree(node)->children.size; i < n; i++) { - TSNode child = ts_node__direct_child(node, i); - if (ts_node_end_byte(child) > max) { - if (ts_node_start_byte(child) > min) break; + TSNode child; + NodeChildIterator iterator = ts_node_child_iterator_begin(&node); + while (ts_node_child_iterator_next(&iterator, &child)) { + if (iterator.position.bytes > max) { + if (child.byte > min) break; node = child; if (ts_node__is_relevant(node, include_anonymous)) last_visible_node = node; did_descend = true; @@ -214,10 +275,13 @@ static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint mi while (did_descend) { did_descend = false; - for (uint32_t i = 0, n = ts_node__tree(node)->children.size; i < n; i++) { - TSNode child = ts_node__direct_child(node, i); + TSNode child; + NodeChildIterator iterator = ts_node_child_iterator_begin(&node); + while (ts_node_child_iterator_next(&iterator, &child)) { const Tree *child_tree = ts_node__tree(child); - if (i > 0) start_position = point_add(start_position, child_tree->padding.extent); + if (iterator.child_index != 1) { + start_position = point_add(start_position, child_tree->padding.extent); + } end_position = point_add(start_position, child_tree->size.extent); if (point_gt(end_position, max)) { if (point_gt(start_position, min)) break; @@ -233,12 +297,10 @@ static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint mi return last_visible_node; } -/* - * Public - */ +// TSNode - Public uint32_t ts_node_start_byte(TSNode self) { - return ts_node__offset_byte(self) + ts_node__tree(self)->padding.bytes; + return self.byte + ts_node__tree(self)->padding.bytes; } uint32_t ts_node_end_byte(TSNode self) { @@ -246,21 +308,16 @@ uint32_t ts_node_end_byte(TSNode self) { } TSPoint ts_node_start_point(TSNode self) { - const Tree *tree = ts_node__tree(self); - return (TSPoint){ ts_node__offset_row(self) + tree->padding.extent.row, - ts_tree_start_column(tree) }; + return point_add(self.position, ts_node__tree(self)->padding.extent); } TSPoint ts_node_end_point(TSNode self) { - const Tree *tree = ts_node__tree(self); - return (TSPoint){ ts_node__offset_row(self) + tree->padding.extent.row + - tree->size.extent.row, - ts_tree_end_column(tree) }; + return point_add(ts_node_start_point(self), ts_node__tree(self)->size.extent); } TSSymbol ts_node_symbol(TSNode self) { const Tree *tree = ts_node__tree(self); - return tree->context.alias_symbol ? tree->context.alias_symbol : tree->symbol; + return self.alias_symbol ? self.alias_symbol : tree->symbol; } const char *ts_node_type(TSNode self, const TSDocument *document) { @@ -272,15 +329,17 @@ char *ts_node_string(TSNode self, const TSDocument *document) { } bool ts_node_eq(TSNode self, TSNode other) { - return + return ( ts_tree_eq(ts_node__tree(self), ts_node__tree(other)) && - self.offset[0] == other.offset[0] && - self.offset[1] == other.offset[1]; + self.byte == other.byte + ); } bool ts_node_is_named(TSNode self) { const Tree *tree = ts_node__tree(self); - return tree->context.alias_symbol ? tree->context.alias_is_named : tree->named; + return self.alias_symbol + ? ts_language_symbol_metadata(self.document->parser.language, self.alias_symbol).named + : tree->named; } bool ts_node_is_missing(TSNode self) { @@ -297,35 +356,31 @@ bool ts_node_has_error(TSNode self) { } TSNode ts_node_parent(TSNode self) { - TSNode result = self; - uint32_t index; + TSNode node = ts_document_root_node(self.document); + uint32_t end_byte = ts_node_end_byte(self); + if (node.subtree == self.subtree) return ts_node__null(); - do { - result = ts_node__direct_parent(result, &index); - if (!result.data) - return ts_node__null(); - } while (!ts_node__tree(result)->visible); + TSNode last_visible_node = node; + bool did_descend = true; + while (did_descend) { + did_descend = false; - return result; -} - -uint32_t ts_node_child_index(TSNode self) { - const Tree *tree = ts_node__tree(self); - uint32_t result = 0; - - for (;;) { - const Tree *parent = tree->context.parent; - uint32_t index = tree->context.index; - if (!parent) return UINT32_MAX; - for (uint32_t i = 0; i < index; i++) { - Tree *child = parent->children.contents[i]; - result += child->visible ? 1 : child->visible_child_count; + TSNode child; + NodeChildIterator iterator = ts_node_child_iterator_begin(&node); + while (ts_node_child_iterator_next(&iterator, &child)) { + if (child.byte > self.byte || child.subtree == self.subtree) break; + if (iterator.position.bytes >= end_byte) { + node = child; + if (ts_node__is_relevant(child, true)) { + last_visible_node = node; + } + did_descend = true; + break; + } } - if (parent->visible) break; - tree = parent; } - return result; + return last_visible_node; } TSNode ts_node_child(TSNode self, uint32_t child_index) { diff --git a/src/runtime/node.h b/src/runtime/node.h deleted file mode 100644 index ee184c9a..00000000 --- a/src/runtime/node.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef RUNTIME_NODE_H_ -#define RUNTIME_NODE_H_ - -#include "runtime/tree.h" - -TSNode ts_node_make(const Tree *, uint32_t byte, uint32_t row); - -#endif diff --git a/src/runtime/parser.c b/src/runtime/parser.c index ff194936..cac6c9e1 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -119,16 +119,19 @@ static bool parser__breakdown_top_of_stack(Parser *self, StackVersion version) { static void parser__breakdown_lookahead(Parser *self, Tree **lookahead, TSStateId state, ReusableNode *reusable_node) { - bool did_break_down = false; - while (reusable_node->tree->children.size > 0 && reusable_node->tree->parse_state != state) { - LOG("state_mismatch sym:%s", SYM_NAME(reusable_node->tree->symbol)); - reusable_node_breakdown(reusable_node); - did_break_down = true; + bool did_descend = false; + Tree *tree = reusable_node_tree(reusable_node); + while (tree->children.size > 0 && tree->parse_state != state) { + LOG("state_mismatch sym:%s", SYM_NAME(tree->symbol)); + reusable_node_descend(reusable_node); + tree = reusable_node_tree(reusable_node); + did_descend = true; } - if (did_break_down) { + if (did_descend) { ts_tree_release(&self->tree_pool, *lookahead); - ts_tree_retain(*lookahead = reusable_node->tree); + *lookahead = tree; + ts_tree_retain(*lookahead); } } @@ -419,21 +422,22 @@ static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId Tree *last_external_token = ts_stack_last_external_token(self->stack, version); Tree *result; - while ((result = reusable_node->tree)) { - if (reusable_node->byte_index > position.bytes) { + while ((result = reusable_node_tree(reusable_node))) { + uint32_t byte_offset = reusable_node_byte_offset(reusable_node); + if (byte_offset > position.bytes) { LOG("before_reusable_node symbol:%s", SYM_NAME(result->symbol)); break; } - if (reusable_node->byte_index < position.bytes) { + if (byte_offset < position.bytes) { LOG("past_reusable_node symbol:%s", SYM_NAME(result->symbol)); - reusable_node_pop(reusable_node); + reusable_node_advance(reusable_node); continue; } if (!ts_tree_external_token_state_eq(reusable_node->last_external_token, last_external_token)) { LOG("reusable_node_has_different_external_scanner_state symbol:%s", SYM_NAME(result->symbol)); - reusable_node_pop(reusable_node); + reusable_node_advance(reusable_node); continue; } @@ -452,8 +456,8 @@ static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId if (reason) { LOG("cant_reuse_node_%s tree:%s", reason, SYM_NAME(result->symbol)); - if (!reusable_node_breakdown(reusable_node)) { - reusable_node_pop(reusable_node); + if (!reusable_node_descend(reusable_node)) { + reusable_node_advance(reusable_node); parser__breakdown_top_of_stack(self, version); *state = ts_stack_state(self->stack, version); } @@ -467,7 +471,7 @@ static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId SYM_NAME(result->symbol), SYM_NAME(result->first_leaf.symbol) ); - reusable_node_pop_leaf(reusable_node); + reusable_node_advance_past_leaf(reusable_node); break; } @@ -678,7 +682,7 @@ static void parser__start(Parser *self, TSInput input, Tree *previous_tree) { ts_lexer_set_input(&self->lexer, input); ts_stack_clear(self->stack); - self->reusable_node = reusable_node_new(previous_tree); + reusable_node_reset(&self->reusable_node, previous_tree); self->finished_tree = NULL; self->accept_count = 0; self->in_ambiguity = false; @@ -1106,7 +1110,9 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re } parser__shift(self, version, next_state, lookahead, action.params.extra); - if (lookahead == reusable_node->tree) reusable_node_pop(reusable_node); + if (lookahead == reusable_node_tree(reusable_node)) { + reusable_node_advance(reusable_node); + } ts_tree_release(&self->tree_pool, lookahead); return; } @@ -1136,7 +1142,9 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re parser__breakdown_lookahead(self, &lookahead, state, reusable_node); } parser__recover(self, version, lookahead); - if (lookahead == reusable_node->tree) reusable_node_pop(reusable_node); + if (lookahead == reusable_node_tree(reusable_node)) { + reusable_node_advance(reusable_node); + } ts_tree_release(&self->tree_pool, lookahead); return; } @@ -1255,6 +1263,7 @@ bool parser_init(Parser *self) { ts_tree_pool_init(&self->tree_pool); self->stack = ts_stack_new(&self->tree_pool); self->finished_tree = NULL; + self->reusable_node = reusable_node_new(); parser__set_cached_token(self, 0, NULL, NULL); return true; } @@ -1277,6 +1286,7 @@ void parser_destroy(Parser *self) { if (self->reduce_actions.contents) array_delete(&self->reduce_actions); ts_tree_pool_delete(&self->tree_pool); + reusable_node_delete(&self->reusable_node); parser_set_language(self, NULL); } @@ -1285,11 +1295,12 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_err StackVersion version = STACK_VERSION_NONE; uint32_t position = 0, last_position = 0; - ReusableNode reusable_node; + ReusableNode reusable_node = reusable_node_new(); + reusable_node_assign(&reusable_node, &self->reusable_node); do { for (version = 0; version < ts_stack_version_count(self->stack); version++) { - reusable_node = self->reusable_node; + reusable_node_assign(&reusable_node, &self->reusable_node); while (ts_stack_is_active(self->stack, version)) { LOG("process version:%d, version_count:%u, state:%d, row:%u, col:%u", @@ -1309,7 +1320,7 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_err } } - self->reusable_node = reusable_node; + reusable_node_assign(&self->reusable_node, &reusable_node); unsigned min_error_cost = parser__condense_stack(self); if (self->finished_tree && self->finished_tree->error_cost < min_error_cost) { @@ -1322,6 +1333,7 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_err self->in_ambiguity = version > 1; } while (version != 0); + reusable_node_delete(&reusable_node); ts_stack_clear(self->stack); parser__set_cached_token(self, 0, NULL, NULL); ts_tree_assign_parents(self->finished_tree, &self->tree_pool, self->language); diff --git a/src/runtime/reusable_node.h b/src/runtime/reusable_node.h index 4a6290f4..803304de 100644 --- a/src/runtime/reusable_node.h +++ b/src/runtime/reusable_node.h @@ -2,44 +2,81 @@ typedef struct { Tree *tree; - uint32_t byte_index; + uint32_t child_index; + uint32_t byte_offset; +} StackEntry; + +typedef struct { + Array(StackEntry) stack; Tree *last_external_token; } ReusableNode; -static inline ReusableNode reusable_node_new(Tree *tree) { - ReusableNode result = {tree, 0, NULL}; - return result; +static inline ReusableNode reusable_node_new() { + return (ReusableNode) {array_new(), NULL}; } -static inline void reusable_node_pop(ReusableNode *self) { - self->byte_index += ts_tree_total_bytes(self->tree); - if (self->tree->has_external_tokens) { - self->last_external_token = ts_tree_last_external_token(self->tree); - } - - while (self->tree) { - Tree *parent = self->tree->context.parent; - uint32_t next_index = self->tree->context.index + 1; - if (parent && parent->children.size > next_index) { - self->tree = parent->children.contents[next_index]; - return; - } - self->tree = parent; - } +static inline void reusable_node_reset(ReusableNode *self, Tree *tree) { + array_clear(&self->stack); + array_push(&self->stack, ((StackEntry) { + .tree = tree, + .child_index = 0, + .byte_offset = 0, + })); } -static inline void reusable_node_pop_leaf(ReusableNode *self) { - while (self->tree->children.size > 0) { - self->tree = self->tree->children.contents[0]; - } - reusable_node_pop(self); +static inline Tree *reusable_node_tree(ReusableNode *self) { + return array_back(&self->stack)->tree; } -static inline bool reusable_node_breakdown(ReusableNode *self) { - if (self->tree->children.size == 0) { - return false; - } else { - self->tree = self->tree->children.contents[0]; +static inline uint32_t reusable_node_byte_offset(ReusableNode *self) { + return array_back(&self->stack)->byte_offset; +} + +static inline void reusable_node_delete(ReusableNode *self) { + array_delete(&self->stack); +} + +static inline void reusable_node_assign(ReusableNode *self, const ReusableNode *other) { + array_assign(&self->stack, &other->stack); +} + +static inline void reusable_node_advance(ReusableNode *self) { + StackEntry last_entry = *array_back(&self->stack); + uint32_t byte_offset = last_entry.byte_offset + ts_tree_total_bytes(last_entry.tree); + if (last_entry.tree->has_external_tokens) { + self->last_external_token = ts_tree_last_external_token(last_entry.tree); + } + + Tree *tree; + uint32_t next_index; + do { + StackEntry popped_entry = array_pop(&self->stack); + next_index = popped_entry.child_index + 1; + tree = array_back(&self->stack)->tree; + } while (tree->children.size <= next_index); + + array_push(&self->stack, ((StackEntry) { + .tree = tree->children.contents[next_index], + .child_index = next_index, + .byte_offset = byte_offset, + })); +} + +static inline bool reusable_node_descend(ReusableNode *self) { + StackEntry last_entry = *array_back(&self->stack); + if (last_entry.tree->children.size > 0) { + array_push(&self->stack, ((StackEntry) { + .tree = last_entry.tree->children.contents[0], + .child_index = 0, + .byte_offset = last_entry.byte_offset, + })); return true; + } else { + return false; } } + +static inline void reusable_node_advance_past_leaf(ReusableNode *self) { + while (reusable_node_descend(self)) {} + reusable_node_advance(self); +} diff --git a/src/runtime/tree.c b/src/runtime/tree.c index c1ada974..fcb534dd 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -169,7 +169,9 @@ Tree *ts_tree_make_copy(TreePool *pool, Tree *self) { return result; } -static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *language) { +static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *language, TreeArray *stack) { + unsigned initial_stack_size = stack->size; + Tree *tree = self; for (unsigned i = 0; i < count; i++) { if (tree->ref_count > 1 || tree->children.size != 2) break; @@ -189,22 +191,14 @@ static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *lang ) break; tree->children.contents[0] = grandchild; - grandchild->context.parent = tree; - grandchild->context.index = -1; - child->children.contents[0] = grandchild->children.contents[1]; - child->children.contents[0]->context.parent = child; - child->children.contents[0]->context.index = -1; - grandchild->children.contents[1] = child; - grandchild->children.contents[1]->context.parent = grandchild; - grandchild->children.contents[1]->context.index = -1; - + array_push(stack, tree); tree = grandchild; } - while (tree != self) { - tree = tree->context.parent; + while (stack->size > initial_stack_size) { + tree = array_pop(stack); Tree *child = tree->children.contents[0]; Tree *grandchild = child->children.contents[1]; ts_tree_set_children(grandchild, &grandchild->children, language); @@ -213,50 +207,30 @@ static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *lang } } -void ts_tree__balance(Tree *self, const TSLanguage *language) { - if (self->children.contents[0]->repeat_depth > self->children.contents[1]->repeat_depth) { - unsigned n = self->children.contents[0]->repeat_depth - self->children.contents[1]->repeat_depth; - for (unsigned i = n / 2; i > 0; i /= 2) { - ts_tree__compress(self, i, language); - n -= i; - } - } -} - void ts_tree_assign_parents(Tree *self, TreePool *pool, const TSLanguage *language) { - self->context.parent = NULL; array_clear(&pool->tree_stack); array_push(&pool->tree_stack, self); while (pool->tree_stack.size > 0) { Tree *tree = array_pop(&pool->tree_stack); if (tree->repeat_depth > 0) { - ts_tree__balance(tree, language); + if (tree->children.contents[0]->repeat_depth > tree->children.contents[1]->repeat_depth) { + unsigned n = ( + tree->children.contents[0]->repeat_depth - + tree->children.contents[1]->repeat_depth + ); + for (unsigned i = n / 2; i > 0; i /= 2) { + ts_tree__compress(tree, i, language, &pool->tree_stack); + n -= i; + } + } } - Length offset = length_zero(); - const TSSymbol *alias_sequence = ts_language_alias_sequence(language, tree->alias_sequence_id); - uint32_t non_extra_index = 0; - bool earlier_child_was_changed = false; for (uint32_t i = 0; i < tree->children.size; i++) { Tree *child = tree->children.contents[i]; - if (earlier_child_was_changed || child->context.parent != tree || child->context.index != i) { - earlier_child_was_changed = true; - child->context.parent = tree; - child->context.index = i; - child->context.offset = offset; - if (!child->extra && alias_sequence && alias_sequence[non_extra_index] != 0) { - TSSymbolMetadata metadata = ts_language_symbol_metadata(language, alias_sequence[non_extra_index]); - child->context.alias_symbol = alias_sequence[non_extra_index]; - child->context.alias_is_named = metadata.named; - } else { - child->context.alias_symbol = 0; - child->context.alias_is_named = false; - } + if (child->ref_count == 1) { array_push(&pool->tree_stack, child); } - offset = length_add(offset, ts_tree_total_size(child)); - if (!child->extra) non_extra_index++; } } } @@ -407,25 +381,6 @@ void ts_tree_release(TreePool *pool, Tree *self) { } } -uint32_t ts_tree_start_column(const Tree *self) { - uint32_t column = self->padding.extent.column; - if (self->padding.extent.row > 0) - return column; - for (const Tree *tree = self; tree != NULL; tree = tree->context.parent) { - column += tree->context.offset.extent.column; - if (tree->context.offset.extent.row > 0) - break; - } - return column; -} - -uint32_t ts_tree_end_column(const Tree *self) { - uint32_t result = self->size.extent.column; - if (self->size.extent.row == 0) - result += ts_tree_start_column(self); - return result; -} - bool ts_tree_eq(const Tree *self, const Tree *other) { if (self) { if (!other) return false; @@ -577,7 +532,6 @@ void ts_tree_edit(Tree *self, const TSInputEdit *edit) { } child_right = length_add(child_left, ts_tree_total_size(child)); - child->context.offset = child_left; } } @@ -612,9 +566,10 @@ static size_t ts_tree__write_char_to_string(char *s, size_t n, int32_t c) { return snprintf(s, n, "%d", c); } -static size_t ts_tree__write_to_string(const Tree *self, const TSLanguage *language, - char *string, size_t limit, bool is_root, - bool include_all) { +static size_t ts_tree__write_to_string(const Tree *self, char *string, size_t limit, + const TSLanguage *language, bool is_root, + bool include_all, TSSymbol alias_symbol, + bool alias_is_named) { if (!self) return snprintf(string, limit, "(NULL)"); char *cursor = string; @@ -624,7 +579,7 @@ static size_t ts_tree__write_to_string(const Tree *self, const TSLanguage *langu is_root || self->is_missing || (self->visible && self->named) || - self->context.alias_is_named; + alias_is_named; if (visible && !is_root) { cursor += snprintf(*writer, limit, " "); @@ -637,15 +592,35 @@ static size_t ts_tree__write_to_string(const Tree *self, const TSLanguage *langu } else if (self->is_missing) { cursor += snprintf(*writer, limit, "(MISSING"); } else { - TSSymbol symbol = self->context.alias_symbol ? self->context.alias_symbol : self->symbol; + TSSymbol symbol = alias_symbol ? alias_symbol : self->symbol; const char *symbol_name = ts_language_symbol_name(language, symbol); cursor += snprintf(*writer, limit, "(%s", symbol_name); } } + const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); + uint32_t structural_child_index = 0; for (uint32_t i = 0; i < self->children.size; i++) { Tree *child = self->children.contents[i]; - cursor += ts_tree__write_to_string(child, language, *writer, limit, false, include_all); + if (child->extra) { + cursor += ts_tree__write_to_string( + child, *writer, limit, + language, false, include_all, + 0, false + ); + } else { + cursor += ts_tree__write_to_string( + child, *writer, limit, + language, false, include_all, + alias_sequence + ? alias_sequence[structural_child_index] + : 0, + alias_sequence + ? ts_language_symbol_metadata(language, alias_sequence[structural_child_index]).named + : false + ); + structural_child_index++; + } } if (visible) cursor += snprintf(*writer, limit, ")"); @@ -655,15 +630,19 @@ static size_t ts_tree__write_to_string(const Tree *self, const TSLanguage *langu char *ts_tree_string(const Tree *self, const TSLanguage *language, bool include_all) { char scratch_string[1]; - size_t size = ts_tree__write_to_string(self, language, scratch_string, 0, true, include_all) + 1; + size_t size = ts_tree__write_to_string( + self, scratch_string, 0, + language, true, + include_all, 0, false + ) + 1; char *result = ts_malloc(size * sizeof(char)); - ts_tree__write_to_string(self, language, result, size, true, include_all); + ts_tree__write_to_string(self, result, size, language, true, include_all, 0, false); return result; } void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset, - const TSLanguage *language, FILE *f) { - TSSymbol symbol = self->context.alias_symbol ? self->context.alias_symbol : self->symbol; + const TSLanguage *language, TSSymbol alias_symbol, FILE *f) { + TSSymbol symbol = alias_symbol ? alias_symbol : self->symbol; fprintf(f, "tree_%p [label=\"%s\"", self, ts_language_symbol_name(language, symbol)); if (self->children.size == 0) @@ -674,9 +653,18 @@ void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset, fprintf(f, ", tooltip=\"address:%p\nrange:%u - %u\nstate:%d\nerror-cost:%u\nrepeat-depth:%u\"]\n", self, byte_offset, byte_offset + ts_tree_total_bytes(self), self->parse_state, self->error_cost, self->repeat_depth); + + const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); + uint32_t structural_child_index = 0; for (uint32_t i = 0; i < self->children.size; i++) { const Tree *child = self->children.contents[i]; - ts_tree__print_dot_graph(child, byte_offset, language, f); + if (child->extra) { + ts_tree__print_dot_graph(child, byte_offset, language, 0, f); + } else { + TSSymbol alias_symbol = alias_sequence ? alias_sequence[structural_child_index] : 0; + ts_tree__print_dot_graph(child, byte_offset, language, alias_symbol, f); + structural_child_index++; + } fprintf(f, "tree_%p -> tree_%p [tooltip=%u]\n", self, child, i); byte_offset += ts_tree_total_bytes(child); } @@ -685,7 +673,7 @@ void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset, void ts_tree_print_dot_graph(const Tree *self, const TSLanguage *language, FILE *f) { fprintf(f, "digraph tree {\n"); fprintf(f, "edge [arrowhead=none]\n"); - ts_tree__print_dot_graph(self, 0, language, f); + ts_tree__print_dot_graph(self, 0, language, 0, f); fprintf(f, "}\n"); } diff --git a/src/runtime/tree.h b/src/runtime/tree.h index f416516d..da5f1e90 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -27,14 +27,6 @@ typedef struct Tree Tree; typedef Array(Tree *) TreeArray; struct Tree { - struct { - struct Tree *parent; - uint32_t index; - Length offset; - TSSymbol alias_symbol : 15; - bool alias_is_named : 1; - } context; - Length padding; Length size; uint32_t ref_count; @@ -106,8 +98,6 @@ void ts_tree_retain(Tree *tree); void ts_tree_release(TreePool *, Tree *tree); bool ts_tree_eq(const Tree *tree1, const Tree *tree2); int ts_tree_compare(const Tree *tree1, const Tree *tree2); -uint32_t ts_tree_start_column(const Tree *self); -uint32_t ts_tree_end_column(const Tree *self); void ts_tree_set_children(Tree *, TreeArray *, const TSLanguage *); void ts_tree_assign_parents(Tree *, TreePool *, const TSLanguage *); void ts_tree_edit(Tree *, const TSInputEdit *edit); @@ -120,6 +110,10 @@ static inline uint32_t ts_tree_total_bytes(const Tree *self) { return self->padding.bytes + self->size.bytes; } +static inline uint32_t ts_tree_total_rows(const Tree *self) { + return self->padding.extent.row + self->size.extent.row; +} + static inline Length ts_tree_total_size(const Tree *self) { return length_add(self->padding, self->size); } diff --git a/test/compiler/prepare_grammar/extract_tokens_test.cc b/test/compiler/prepare_grammar/extract_tokens_test.cc index 0f9be780..da2ae60a 100644 --- a/test/compiler/prepare_grammar/extract_tokens_test.cc +++ b/test/compiler/prepare_grammar/extract_tokens_test.cc @@ -48,7 +48,7 @@ describe("extract_tokens", []() { Repeat{Symbol::non_terminal(3)} }, }, - {}, {}, {}, {} + {}, {}, {}, {}, {} }); InitialSyntaxGrammar &syntax_grammar = get<0>(result); @@ -156,7 +156,7 @@ describe("extract_tokens", []() { }) }, }, - {}, {}, {}, {} + {}, {}, {}, {}, {} }); InitialSyntaxGrammar &syntax_grammar = get<0>(result); @@ -203,7 +203,7 @@ describe("extract_tokens", []() { Rule::seq({ String{"ef"}, String{"cd"} }) }, }, - {}, {}, {}, {} + {}, {}, {}, {}, {} }); InitialSyntaxGrammar &syntax_grammar = get<0>(result); @@ -258,7 +258,7 @@ describe("extract_tokens", []() { String{"a"} }, }, - {}, {}, {}, {} + {}, {}, {}, {}, {} }); InitialSyntaxGrammar &syntax_grammar = get<0>(result); @@ -298,7 +298,7 @@ describe("extract_tokens", []() { { { Symbol::non_terminal(2), Symbol::non_terminal(3) } }, - {}, {} + {}, {}, {} }); InitialSyntaxGrammar &syntax_grammar = get<0>(result); @@ -319,7 +319,7 @@ describe("extract_tokens", []() { String{"y"}, Pattern{" "}, }, - {}, {}, {} + {}, {}, {}, {} }); AssertThat(get<2>(result), Equals(CompileError::none())); @@ -340,7 +340,7 @@ describe("extract_tokens", []() { { String{"y"}, }, - {}, {}, {} + {}, {}, {}, {} }); AssertThat(get<2>(result), Equals(CompileError::none())); @@ -370,7 +370,7 @@ describe("extract_tokens", []() { { Symbol::non_terminal(2), }, - {}, {}, {} + {}, {}, {}, {} }); AssertThat(get<2>(result), Equals(CompileError::none())); @@ -399,7 +399,7 @@ describe("extract_tokens", []() { { Symbol::non_terminal(1) }, - {}, {}, {} + {}, {}, {}, {} }); AssertThat(get<2>(result), Equals(CompileError( @@ -417,7 +417,7 @@ describe("extract_tokens", []() { { Rule::choice({ Symbol::non_terminal(1), Blank{} }) }, - {}, {}, {} + {}, {}, {}, {} }); AssertThat(get<2>(result), Equals(CompileError( @@ -446,7 +446,7 @@ describe("extract_tokens", []() { { Variable{"rule_A", VariableTypeNamed, Symbol::non_terminal(0)} }, - {} + {}, {} }); AssertThat(get<2>(result), Equals(CompileError( diff --git a/test/helpers/tree_helpers.cc b/test/helpers/tree_helpers.cc index 5142ca8d..8fedb560 100644 --- a/test/helpers/tree_helpers.cc +++ b/test/helpers/tree_helpers.cc @@ -1,8 +1,7 @@ -#include "bandit/bandit.h" +#include "test_helper.h" #include "helpers/tree_helpers.h" #include "helpers/point_helpers.h" #include "runtime/document.h" -#include "runtime/node.h" #include using std::string; @@ -21,24 +20,30 @@ TreeArray *tree_array(std::vector trees) { result.capacity = trees.size(); result.size = trees.size(); result.contents = (Tree **)calloc(trees.size(), sizeof(Tree *)); - for (size_t i = 0; i < trees.size(); i++) + for (size_t i = 0; i < trees.size(); i++) { result.contents[i] = trees[i]; + } return &result; } ostream &operator<<(std::ostream &stream, const Tree *tree) { static TSLanguage DUMMY_LANGUAGE = {}; - static TSDocument DUMMY_DOCUMENT = {}; - DUMMY_DOCUMENT.parser.language = &DUMMY_LANGUAGE; DUMMY_LANGUAGE.symbol_names = symbol_names; - TSNode node; - node.data = tree; - return stream << string(ts_node_string(node, &DUMMY_DOCUMENT)); + char *string = ts_tree_string(tree, &DUMMY_LANGUAGE, false); + stream << string; + ts_free(string); + return stream; } ostream &operator<<(ostream &stream, const TSNode &node) { - return stream << string("{") << (const Tree *)node.data << - string(", ") << to_string(ts_node_start_byte(node)) << string("}"); + if (node.subtree) { + char *string = ts_node_string(node, node.document); + stream << "{" << string << ", " << to_string(ts_node_start_byte(node)) << "}"; + ts_free(string); + return stream; + } else { + return stream << "NULL"; + } } bool operator==(const TSNode &left, const TSNode &right) { diff --git a/test/runtime/document_test.cc b/test/runtime/document_test.cc index 0be03657..97aa4fe2 100644 --- a/test/runtime/document_test.cc +++ b/test/runtime/document_test.cc @@ -182,7 +182,7 @@ describe("Document", [&]() { ts_document_parse(document); ts_document_set_language(document, load_real_language("javascript")); - AssertThat(ts_document_root_node(document).data, Equals(nullptr)); + AssertThat(ts_document_root_node(document).subtree, Equals(nullptr)); ts_document_parse(document); root = ts_document_root_node(document); diff --git a/test/runtime/node_test.cc b/test/runtime/node_test.cc index ff12169f..22196ec5 100644 --- a/test/runtime/node_test.cc +++ b/test/runtime/node_test.cc @@ -71,6 +71,7 @@ describe("Node", [&]() { document = ts_document_new(); ts_document_set_language(document, load_real_language("json")); ts_document_set_input_string(document, json_string.c_str()); + // ts_document_print_debugging_graphs(document, true); ts_document_parse(document); root_node = ts_node_child(ts_document_root_node(document), 0); }); @@ -157,7 +158,7 @@ describe("Node", [&]() { AssertThat(ts_node_parent(number_node), Equals(root_node)); AssertThat(ts_node_parent(false_node), Equals(root_node)); AssertThat(ts_node_parent(object_node), Equals(root_node)); - AssertThat(ts_node_parent(ts_document_root_node(document)).data, Equals(nullptr)); + AssertThat(ts_node_parent(ts_document_root_node(document)).subtree, Equals(nullptr)); }); it("works correctly when the node contains aliased children and extras", [&]() { @@ -239,19 +240,7 @@ describe("Node", [&]() { child = ts_node_first_named_child_for_byte(root_node, object_index + 1); AssertThat(ts_node_type(child, document), Equals("object")); child = ts_node_first_named_child_for_byte(root_node, object_end_index); - AssertThat(child.data, Equals(nullptr)); - }); - }); - - describe("child_index()", [&]() { - it("returns the index of the node within its parent", [&]() { - AssertThat(ts_node_child_index(ts_node_child(root_node, 0)), Equals(0u)); - AssertThat(ts_node_child_index(ts_node_child(root_node, 1)), Equals(1u)); - AssertThat(ts_node_child_index(ts_node_child(root_node, 2)), Equals(2u)); - AssertThat(ts_node_child_index(ts_node_child(root_node, 3)), Equals(3u)); - AssertThat(ts_node_child_index(ts_node_child(root_node, 4)), Equals(4u)); - AssertThat(ts_node_child_index(ts_node_child(root_node, 5)), Equals(5u)); - AssertThat(ts_node_child_index(ts_node_child(root_node, 6)), Equals(6u)); + AssertThat(child.subtree, Equals(nullptr)); }); }); @@ -335,7 +324,7 @@ describe("Node", [&]() { AssertThat(ts_node_parent(child5), Equals(root_node)); AssertThat(ts_node_parent(child6), Equals(root_node)); AssertThat(ts_node_parent(child7), Equals(root_node)); - AssertThat(ts_node_parent(ts_document_root_node(document)).data, Equals(nullptr)); + AssertThat(ts_node_parent(ts_document_root_node(document)).subtree, Equals(nullptr)); }); }); @@ -355,15 +344,16 @@ describe("Node", [&]() { TSNode brace_node2 = ts_node_child(object_node, 2); TSNode bracket_node2 = ts_node_child(root_node, 6); + AssertThat(ts_node_parent(bracket_node1), Equals(root_node)); AssertThat(ts_node_next_sibling(bracket_node1), Equals(number_node)); AssertThat(ts_node_next_sibling(number_node), Equals(array_comma_node1)); AssertThat(ts_node_next_sibling(array_comma_node1), Equals(false_node)); AssertThat(ts_node_next_sibling(false_node), Equals(array_comma_node2)); AssertThat(ts_node_next_sibling(array_comma_node2), Equals(object_node)); AssertThat(ts_node_next_sibling(object_node), Equals(bracket_node2)); - AssertThat(ts_node_next_sibling(bracket_node2).data, Equals(nullptr)); + AssertThat(ts_node_next_sibling(bracket_node2).subtree, Equals(nullptr)); - AssertThat(ts_node_prev_sibling(bracket_node1).data, Equals(nullptr)); + AssertThat(ts_node_prev_sibling(bracket_node1).subtree, Equals(nullptr)); AssertThat(ts_node_prev_sibling(number_node), Equals(bracket_node1)); AssertThat(ts_node_prev_sibling(array_comma_node1), Equals(number_node)); AssertThat(ts_node_prev_sibling(false_node), Equals(array_comma_node1)); @@ -373,24 +363,24 @@ describe("Node", [&]() { AssertThat(ts_node_next_sibling(brace_node1), Equals(pair_node)); AssertThat(ts_node_next_sibling(pair_node), Equals(brace_node2)); - AssertThat(ts_node_next_sibling(brace_node2).data, Equals(nullptr)); + AssertThat(ts_node_next_sibling(brace_node2).subtree, Equals(nullptr)); - AssertThat(ts_node_prev_sibling(brace_node1).data, Equals(nullptr)); + AssertThat(ts_node_prev_sibling(brace_node1).subtree, Equals(nullptr)); AssertThat(ts_node_prev_sibling(pair_node), Equals(brace_node1)); AssertThat(ts_node_prev_sibling(brace_node2), Equals(pair_node)); AssertThat(ts_node_next_sibling(string_node), Equals(colon_node)); AssertThat(ts_node_next_sibling(colon_node), Equals(null_node)); - AssertThat(ts_node_next_sibling(null_node).data, Equals(nullptr)); + AssertThat(ts_node_next_sibling(null_node).subtree, Equals(nullptr)); - AssertThat(ts_node_prev_sibling(string_node).data, Equals(nullptr)); + AssertThat(ts_node_prev_sibling(string_node).subtree, Equals(nullptr)); AssertThat(ts_node_prev_sibling(colon_node), Equals(string_node)); AssertThat(ts_node_prev_sibling(null_node), Equals(colon_node)); }); it("returns null when the node has no parent", [&]() { - AssertThat(ts_node_next_named_sibling(root_node).data, Equals(nullptr)); - AssertThat(ts_node_prev_named_sibling(root_node).data, Equals(nullptr)); + AssertThat(ts_node_next_named_sibling(root_node).subtree, Equals(nullptr)); + AssertThat(ts_node_prev_named_sibling(root_node).subtree, Equals(nullptr)); }); }); @@ -412,8 +402,8 @@ describe("Node", [&]() { }); it("returns null when the node has no parent", [&]() { - AssertThat(ts_node_next_named_sibling(root_node).data, Equals(nullptr)); - AssertThat(ts_node_prev_named_sibling(root_node).data, Equals(nullptr)); + AssertThat(ts_node_next_named_sibling(root_node).subtree, Equals(nullptr)); + AssertThat(ts_node_prev_named_sibling(root_node).subtree, Equals(nullptr)); }); }); diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc index 8669b6c1..262572b1 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/tree_test.cc @@ -5,14 +5,12 @@ #include "runtime/length.h" void assert_consistent(const Tree *tree) { - if (tree->child_count == 0) - return; + if (tree->child_count == 0) return; AssertThat(tree->children.contents[0]->padding, Equals(tree->padding)); Length total_children_size = length_zero(); for (size_t i = 0; i < tree->children.size; i++) { Tree *child = tree->children.contents[i]; - AssertThat(child->context.offset, Equals(total_children_size)); assert_consistent(child); total_children_size = length_add(total_children_size, ts_tree_total_size(child)); } From f857d64d54621f0ef74c92f1d235b8e93f2c2cd1 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 13:23:47 -0700 Subject: [PATCH 04/31] Avoid recursion in ts_node__prev_sibling --- src/runtime/node.c | 91 +++++++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/src/runtime/node.c b/src/runtime/node.c index 246cd290..2230d1af 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -129,54 +129,53 @@ static inline TSNode ts_node__child(TSNode self, uint32_t child_index, bool incl return ts_node__null(); } -static inline bool ts_node__last_child_before(TSNode self, TSNode target, - bool include_anonymous, TSNode *result) { - TSNode child; - TSNode earlier_child = ts_node__null(); - bool earlier_child_is_relevant = false; - bool found_child_containing_target = false; - NodeChildIterator iterator = ts_node_child_iterator_begin(&self); - uint32_t target_end_byte = ts_node_end_byte(target); - while (ts_node_child_iterator_next(&iterator, &child)) { - if (iterator.position.bytes >= target_end_byte) { - found_child_containing_target = true; - break; - } - - if (ts_node__is_relevant(child, include_anonymous)) { - earlier_child = child; - earlier_child_is_relevant = true; - } else if (ts_node__relevant_child_count(child, include_anonymous) > 0) { - earlier_child = child; - earlier_child_is_relevant = false; - } - } - - if (found_child_containing_target && child.subtree != target.subtree) { - if (ts_node__last_child_before(child, target, include_anonymous, result)) { - return true; - } - } - - if (earlier_child_is_relevant) { - *result = earlier_child; - return true; - } - - if (earlier_child.subtree) { - return ts_node__last_child_before(earlier_child, target, include_anonymous, result); - } - - return false; -} - static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) { - TSNode result = ts_node__null(); - TSNode parent = ts_node_parent(self); - if (parent.subtree) { - ts_node__last_child_before(parent, self, include_anonymous, &result); + uint32_t target_end_byte = ts_node_end_byte(self); + + TSNode node = ts_node_parent(self); + TSNode earlier_node = ts_node__null(); + bool earlier_node_is_relevant = false; + + while (node.subtree) { + TSNode earlier_child = ts_node__null(); + bool earlier_child_is_relevant = false; + bool found_child_containing_target = false; + + TSNode child; + NodeChildIterator iterator = ts_node_child_iterator_begin(&node); + while (ts_node_child_iterator_next(&iterator, &child)) { + if (iterator.position.bytes >= target_end_byte) { + found_child_containing_target = child.subtree != self.subtree; + break; + } + + if (ts_node__is_relevant(child, include_anonymous)) { + earlier_child = child; + earlier_child_is_relevant = true; + } else if (ts_node__relevant_child_count(child, include_anonymous) > 0) { + earlier_child = child; + earlier_child_is_relevant = false; + } + } + + if (found_child_containing_target) { + if (earlier_child.subtree) { + earlier_node = earlier_child; + earlier_node_is_relevant = earlier_child_is_relevant; + } + node = child; + } else if (earlier_child_is_relevant) { + return earlier_child; + } else if (earlier_child.subtree) { + node = earlier_child; + } else if (earlier_node_is_relevant) { + return earlier_node; + } else { + node = earlier_node; + } } - return result; + + return ts_node__null(); } static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) { From 19e3750f13634d2820f5852fb135c0c0f1e20e0d Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 13:46:46 -0700 Subject: [PATCH 05/31] Make ts_node_next_sibling work more like ts_node_prev_sibling Co-Authored-By: Rick Winfrey --- src/runtime/node.c | 51 +++++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/src/runtime/node.c b/src/runtime/node.c index 2230d1af..99b6256d 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -179,28 +179,51 @@ static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) } static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) { - TSNode node = ts_node_parent(self); - if (!node.subtree) return ts_node__null(); - uint32_t end_byte = ts_node_end_byte(self); + uint32_t target_end_byte = ts_node_end_byte(self); - bool did_descend = true; - while (did_descend) { - did_descend = false; + TSNode node = ts_node_parent(self); + TSNode later_node = ts_node__null(); + bool later_node_is_relevant = false; + + while (node.subtree) { + TSNode later_child = ts_node__null(); + bool later_child_is_relevant = false; + TSNode child_containing_target = ts_node__null(); TSNode child; NodeChildIterator iterator = ts_node_child_iterator_begin(&node); while (ts_node_child_iterator_next(&iterator, &child)) { - if (iterator.position.bytes > end_byte && child.subtree != self.subtree) { - if (ts_node__is_relevant(child, include_anonymous)) { - return child; - } - if (ts_node__relevant_child_count(child, include_anonymous) > 0) { - node = child; - did_descend = true; - break; + if (iterator.position.bytes < target_end_byte) continue; + if (child.byte <= self.byte) { + if (child.subtree != self.subtree) { + child_containing_target = child; } + } else if (ts_node__is_relevant(child, include_anonymous)) { + later_child = child; + later_child_is_relevant = true; + break; + } else if (ts_node__relevant_child_count(child, include_anonymous) > 0) { + later_child = child; + later_child_is_relevant = false; + break; } } + + if (child_containing_target.subtree) { + if (later_child.subtree) { + later_node = later_child; + later_node_is_relevant = later_child_is_relevant; + } + node = child_containing_target; + } else if (later_child_is_relevant) { + return later_child; + } else if (later_child.subtree) { + node = later_child; + } else if (later_node_is_relevant) { + return later_node; + } else { + node = later_node; + } } return ts_node__null(); From b06747b6ca01a868102af30e5db8dc76ac8620cf Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 14:14:42 -0700 Subject: [PATCH 06/31] Remove stale unit tests Co-Authored-By: Rick Winfrey --- test/runtime/parser_test.cc | 47 ------------------------------------- 1 file changed, 47 deletions(-) diff --git a/test/runtime/parser_test.cc b/test/runtime/parser_test.cc index 4481ece7..204eedbd 100644 --- a/test/runtime/parser_test.cc +++ b/test/runtime/parser_test.cc @@ -206,53 +206,6 @@ describe("Parser", [&]() { }); }); - describe("handling extra tokens", [&]() { - describe("when the token appears as part of a grammar rule", [&]() { - it("incorporates it into the tree", [&]() { - ts_document_set_language(document, load_real_language("javascript")); - set_text("fn()\n"); - - assert_root_node( - "(program (expression_statement (call_expression (identifier) (arguments))))"); - }); - }); - - describe("when the token appears somewhere else", [&]() { - it("incorporates it into the tree", [&]() { - ts_document_set_language(document, load_real_language("javascript")); - set_text( - "fn()\n" - " .otherFn();"); - - assert_root_node( - "(program (expression_statement (call_expression " - "(member_expression " - "(call_expression (identifier) (arguments)) " - "(property_identifier)) " - "(arguments))))"); - }); - }); - - describe("when several extra tokens appear in a row", [&]() { - it("incorporates them into the tree", [&]() { - ts_document_set_language(document, load_real_language("javascript")); - set_text( - "fn()\n\n" - "// This is a comment" - "\n\n" - ".otherFn();"); - - assert_root_node( - "(program (expression_statement (call_expression " - "(member_expression " - "(call_expression (identifier) (arguments)) " - "(comment) " - "(property_identifier)) " - "(arguments))))"); - }); - }); - }); - describe("editing", [&]() { describe("creating new tokens near the end of the input", [&]() { it("updates the parse tree and re-reads only the changed portion of the text", [&]() { From 5fa6d395785129c65e271f8b321846f239e9b4e4 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 14:14:56 -0700 Subject: [PATCH 07/31] Fix alias handling in ts_tree_string Co-Authored-By: Rick Winfrey --- src/runtime/tree.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/runtime/tree.c b/src/runtime/tree.c index fcb534dd..0174d3d8 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -609,15 +609,12 @@ static size_t ts_tree__write_to_string(const Tree *self, char *string, size_t li 0, false ); } else { + TSSymbol alias_symbol = alias_sequence ? alias_sequence[structural_child_index] : 0; cursor += ts_tree__write_to_string( child, *writer, limit, language, false, include_all, - alias_sequence - ? alias_sequence[structural_child_index] - : 0, - alias_sequence - ? ts_language_symbol_metadata(language, alias_sequence[structural_child_index]).named - : false + alias_symbol, + alias_symbol ? ts_language_symbol_metadata(language, alias_symbol).named : false ); structural_child_index++; } From 8d805feab16423e0ee51b818eef32c93b924182b Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 14:15:11 -0700 Subject: [PATCH 08/31] Allow ReusableNode to advance off the end of the tree Co-Authored-By: Rick Winfrey --- src/runtime/reusable_node.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/runtime/reusable_node.h b/src/runtime/reusable_node.h index 803304de..71d56b68 100644 --- a/src/runtime/reusable_node.h +++ b/src/runtime/reusable_node.h @@ -25,11 +25,15 @@ static inline void reusable_node_reset(ReusableNode *self, Tree *tree) { } static inline Tree *reusable_node_tree(ReusableNode *self) { - return array_back(&self->stack)->tree; + return self->stack.size > 0 + ? self->stack.contents[self->stack.size - 1].tree + : NULL; } static inline uint32_t reusable_node_byte_offset(ReusableNode *self) { - return array_back(&self->stack)->byte_offset; + return self->stack.size > 0 + ? self->stack.contents[self->stack.size - 1].byte_offset + : UINT32_MAX; } static inline void reusable_node_delete(ReusableNode *self) { @@ -52,6 +56,7 @@ static inline void reusable_node_advance(ReusableNode *self) { do { StackEntry popped_entry = array_pop(&self->stack); next_index = popped_entry.child_index + 1; + if (self->stack.size == 0) return; tree = array_back(&self->stack)->tree; } while (tree->children.size <= next_index); From 92255bbfdda33970594b2f6a6f573221e333063c Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 15:28:28 -0700 Subject: [PATCH 09/31] Remove document parameter from ts_node_type, ts_node_string Co-Authored-By: Rick Winfrey --- include/tree_sitter/runtime.h | 4 +- src/runtime/node.c | 8 +- test/helpers/scope_sequence.cc | 2 +- test/helpers/tree_helpers.cc | 2 +- test/integration/real_grammars.cc | 2 +- test/integration/test_grammars.cc | 2 +- test/runtime/document_test.cc | 2 +- test/runtime/language_test.cc | 4 +- test/runtime/node_test.cc | 125 +++++++++++++++--------------- test/runtime/parser_test.cc | 24 +++--- 10 files changed, 87 insertions(+), 88 deletions(-) diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index c4bc8ab5..934d5320 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -75,8 +75,8 @@ TSPoint ts_node_start_point(TSNode); uint32_t ts_node_end_byte(TSNode); TSPoint ts_node_end_point(TSNode); TSSymbol ts_node_symbol(TSNode); -const char *ts_node_type(TSNode, const TSDocument *); -char *ts_node_string(TSNode, const TSDocument *); +const char *ts_node_type(TSNode); +char *ts_node_string(TSNode); bool ts_node_eq(TSNode, TSNode); bool ts_node_is_named(TSNode); bool ts_node_is_missing(TSNode); diff --git a/src/runtime/node.c b/src/runtime/node.c index 99b6256d..77d913f3 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -342,12 +342,12 @@ TSSymbol ts_node_symbol(TSNode self) { return self.alias_symbol ? self.alias_symbol : tree->symbol; } -const char *ts_node_type(TSNode self, const TSDocument *document) { - return ts_language_symbol_name(document->parser.language, ts_node_symbol(self)); +const char *ts_node_type(TSNode self) { + return ts_language_symbol_name(self.document->parser.language, ts_node_symbol(self)); } -char *ts_node_string(TSNode self, const TSDocument *document) { - return ts_tree_string(ts_node__tree(self), document->parser.language, false); +char *ts_node_string(TSNode self) { + return ts_tree_string(ts_node__tree(self), self.document->parser.language, false); } bool ts_node_eq(TSNode self, TSNode other) { diff --git a/test/helpers/scope_sequence.cc b/test/helpers/scope_sequence.cc index d6e2e3b1..c3db70ac 100644 --- a/test/helpers/scope_sequence.cc +++ b/test/helpers/scope_sequence.cc @@ -27,7 +27,7 @@ static void append_to_scope_sequence(ScopeSequence *sequence, sequence, current_scopes, text, ts_node_start_byte(node) - sequence->size() ); - current_scopes->push_back(ts_node_type(node, document)); + current_scopes->push_back(ts_node_type(node)); for (size_t i = 0, n = ts_node_child_count(node); i < n; i++) { TSNode child = ts_node_child(node, i); diff --git a/test/helpers/tree_helpers.cc b/test/helpers/tree_helpers.cc index 8fedb560..553f8b49 100644 --- a/test/helpers/tree_helpers.cc +++ b/test/helpers/tree_helpers.cc @@ -37,7 +37,7 @@ ostream &operator<<(std::ostream &stream, const Tree *tree) { ostream &operator<<(ostream &stream, const TSNode &node) { if (node.subtree) { - char *string = ts_node_string(node, node.document); + char *string = ts_node_string(node); stream << "{" << string << ", " << to_string(ts_node_start_byte(node)) << "}"; ts_free(string); return stream; diff --git a/test/integration/real_grammars.cc b/test/integration/real_grammars.cc index e842ee0c..608ded82 100644 --- a/test/integration/real_grammars.cc +++ b/test/integration/real_grammars.cc @@ -63,7 +63,7 @@ for (auto &language_name : test_languages) { edit_sequence(); TSNode root_node = ts_document_root_node(document); - const char *node_string = ts_node_string(root_node, document); + const char *node_string = ts_node_string(root_node); string result(node_string); ts_free((void *)node_string); AssertThat(result, Equals(entry.tree_string)); diff --git a/test/integration/test_grammars.cc b/test/integration/test_grammars.cc index f4ba6d92..f5324579 100644 --- a/test/integration/test_grammars.cc +++ b/test/integration/test_grammars.cc @@ -66,7 +66,7 @@ for (auto &language_name : test_languages) { TSNode root_node = ts_document_root_node(document); AssertThat(ts_node_end_byte(root_node), Equals(entry.input.size())); assert_consistent_tree_sizes(root_node); - const char *node_string = ts_node_string(root_node, document); + const char *node_string = ts_node_string(root_node); string result(node_string); ts_free((void *)node_string); ts_document_free(document); diff --git a/test/runtime/document_test.cc b/test/runtime/document_test.cc index 97aa4fe2..5f140251 100644 --- a/test/runtime/document_test.cc +++ b/test/runtime/document_test.cc @@ -35,7 +35,7 @@ describe("Document", [&]() { }); auto assert_node_string_equals = [&](TSNode node, const string &expected) { - char *str = ts_node_string(node, document); + char *str = ts_node_string(node); string actual(str); ts_free(str); AssertThat(actual, Equals(expected)); diff --git a/test/runtime/language_test.cc b/test/runtime/language_test.cc index 7d9d51cc..4726c4ba 100644 --- a/test/runtime/language_test.cc +++ b/test/runtime/language_test.cc @@ -35,11 +35,11 @@ describe("Language", []() { ts_document_parse(document); TSNode root_node = ts_document_root_node(document); - char *string = ts_node_string(root_node, document); + char *string = ts_node_string(root_node); AssertThat(string, Equals("(a (c))")); TSNode aliased_node = ts_node_child(root_node, 0); - AssertThat(ts_node_type(aliased_node, document), Equals("c")); + AssertThat(ts_node_type(aliased_node), Equals("c")); TSSymbol aliased_symbol = ts_node_symbol(aliased_node); AssertThat(ts_language_symbol_count(language), IsGreaterThan(aliased_symbol)); diff --git a/test/runtime/node_test.cc b/test/runtime/node_test.cc index 22196ec5..91717ed2 100644 --- a/test/runtime/node_test.cc +++ b/test/runtime/node_test.cc @@ -71,7 +71,6 @@ describe("Node", [&]() { document = ts_document_new(); ts_document_set_language(document, load_real_language("json")); ts_document_set_input_string(document, json_string.c_str()); - // ts_document_print_debugging_graphs(document, true); ts_document_parse(document); root_node = ts_node_child(ts_document_root_node(document), 0); }); @@ -84,7 +83,7 @@ describe("Node", [&]() { }); it("parses the example as expected (precondition)", [&]() { - char *node_string = ts_node_string(root_node, document); + char *node_string = ts_node_string(root_node); AssertThat(node_string, Equals( "(array " "(number) " @@ -95,7 +94,7 @@ describe("Node", [&]() { describe("named_child_count(), named_child(i)", [&]() { it("returns the named child node at the given index", [&]() { - AssertThat(ts_node_type(root_node, document), Equals("array")); + AssertThat(ts_node_type(root_node), Equals("array")); AssertThat(ts_node_named_child_count(root_node), Equals(3)); AssertThat(ts_node_start_byte(root_node), Equals(array_index)); @@ -107,9 +106,9 @@ describe("Node", [&]() { TSNode false_node = ts_node_named_child(root_node, 1); TSNode object_node = ts_node_named_child(root_node, 2); - AssertThat(ts_node_type(number_node, document), Equals("number")); - AssertThat(ts_node_type(false_node, document), Equals("false")); - AssertThat(ts_node_type(object_node, document), Equals("object")); + AssertThat(ts_node_type(number_node), Equals("number")); + AssertThat(ts_node_type(false_node), Equals("false")); + AssertThat(ts_node_type(object_node), Equals("object")); AssertThat(ts_node_start_byte(number_node), Equals(number_index)); AssertThat(ts_node_end_byte(number_node), Equals(number_end_index)); @@ -129,7 +128,7 @@ describe("Node", [&]() { TSNode pair_node = ts_node_named_child(object_node, 0); - AssertThat(ts_node_type(pair_node, document), Equals("pair")); + AssertThat(ts_node_type(pair_node), Equals("pair")); AssertThat(ts_node_start_byte(pair_node), Equals(string_index)); AssertThat(ts_node_end_byte(pair_node), Equals(null_end_index)); AssertThat(ts_node_start_point(pair_node), Equals({ 6, 4 })); @@ -139,8 +138,8 @@ describe("Node", [&]() { TSNode string_node = ts_node_named_child(pair_node, 0); TSNode null_node = ts_node_named_child(pair_node, 1); - AssertThat(ts_node_type(string_node, document), Equals("string")); - AssertThat(ts_node_type(null_node, document), Equals("null")); + AssertThat(ts_node_type(string_node), Equals("string")); + AssertThat(ts_node_type(null_node), Equals("null")); AssertThat(ts_node_start_byte(string_node), Equals(string_index)); AssertThat(ts_node_end_byte(string_node), Equals(string_end_index)); @@ -169,16 +168,16 @@ describe("Node", [&]() { ts_document_parse(document); root_node = ts_document_root_node(document); - char *node_string = ts_node_string(root_node, document); + char *node_string = ts_node_string(root_node); AssertThat(node_string, Equals("(a (b) (comment) (B) (comment) (b))")); ts_free(node_string); AssertThat(ts_node_named_child_count(root_node), Equals(5u)); - AssertThat(ts_node_type(ts_node_named_child(root_node, 0), document), Equals("b")); - AssertThat(ts_node_type(ts_node_named_child(root_node, 1), document), Equals("comment")); - AssertThat(ts_node_type(ts_node_named_child(root_node, 2), document), Equals("B")); - AssertThat(ts_node_type(ts_node_named_child(root_node, 3), document), Equals("comment")); - AssertThat(ts_node_type(ts_node_named_child(root_node, 4), document), Equals("b")); + AssertThat(ts_node_type(ts_node_named_child(root_node, 0)), Equals("b")); + AssertThat(ts_node_type(ts_node_named_child(root_node, 1)), Equals("comment")); + AssertThat(ts_node_type(ts_node_named_child(root_node, 2)), Equals("B")); + AssertThat(ts_node_type(ts_node_named_child(root_node, 3)), Equals("comment")); + AssertThat(ts_node_type(ts_node_named_child(root_node, 4)), Equals("b")); AssertThat(ts_node_symbol(ts_node_named_child(root_node, 0)), !Equals(ts_node_symbol(ts_node_named_child(root_node, 2)))); }); @@ -189,29 +188,29 @@ describe("Node", [&]() { TSNode child; child = ts_node_first_child_for_byte(root_node, array_index); - AssertThat(ts_node_type(child, document), Equals("[")); + AssertThat(ts_node_type(child), Equals("[")); child = ts_node_first_child_for_byte(root_node, number_index); - AssertThat(ts_node_type(child, document), Equals("number")); + AssertThat(ts_node_type(child), Equals("number")); child = ts_node_first_child_for_byte(root_node, number_end_index); - AssertThat(ts_node_type(child, document), Equals(",")); + AssertThat(ts_node_type(child), Equals(",")); child = ts_node_first_child_for_byte(root_node, number_end_index + 1); - AssertThat(ts_node_type(child, document), Equals("false")); + AssertThat(ts_node_type(child), Equals("false")); child = ts_node_first_child_for_byte(root_node, false_index - 1); - AssertThat(ts_node_type(child, document), Equals("false")); + AssertThat(ts_node_type(child), Equals("false")); child = ts_node_first_child_for_byte(root_node, false_index); - AssertThat(ts_node_type(child, document), Equals("false")); + AssertThat(ts_node_type(child), Equals("false")); child = ts_node_first_child_for_byte(root_node, false_index + 1); - AssertThat(ts_node_type(child, document), Equals("false")); + AssertThat(ts_node_type(child), Equals("false")); child = ts_node_first_child_for_byte(root_node, false_end_index); - AssertThat(ts_node_type(child, document), Equals(",")); + AssertThat(ts_node_type(child), Equals(",")); child = ts_node_first_child_for_byte(root_node, false_end_index); - AssertThat(ts_node_type(child, document), Equals(",")); + AssertThat(ts_node_type(child), Equals(",")); child = ts_node_first_child_for_byte(root_node, object_index); - AssertThat(ts_node_type(child, document), Equals("object")); + AssertThat(ts_node_type(child), Equals("object")); child = ts_node_first_child_for_byte(root_node, object_index + 1); - AssertThat(ts_node_type(child, document), Equals("object")); + AssertThat(ts_node_type(child), Equals("object")); child = ts_node_first_child_for_byte(root_node, object_end_index); - AssertThat(ts_node_type(child, document), Equals("]")); + AssertThat(ts_node_type(child), Equals("]")); }); }); @@ -220,25 +219,25 @@ describe("Node", [&]() { TSNode child; child = ts_node_first_named_child_for_byte(root_node, array_index); - AssertThat(ts_node_type(child, document), Equals("number")); + AssertThat(ts_node_type(child), Equals("number")); child = ts_node_first_named_child_for_byte(root_node, number_index); - AssertThat(ts_node_type(child, document), Equals("number")); + AssertThat(ts_node_type(child), Equals("number")); child = ts_node_first_named_child_for_byte(root_node, number_end_index); - AssertThat(ts_node_type(child, document), Equals("false")); + AssertThat(ts_node_type(child), Equals("false")); child = ts_node_first_named_child_for_byte(root_node, number_end_index + 1); - AssertThat(ts_node_type(child, document), Equals("false")); + AssertThat(ts_node_type(child), Equals("false")); child = ts_node_first_named_child_for_byte(root_node, false_index - 1); - AssertThat(ts_node_type(child, document), Equals("false")); + AssertThat(ts_node_type(child), Equals("false")); child = ts_node_first_named_child_for_byte(root_node, false_index); - AssertThat(ts_node_type(child, document), Equals("false")); + AssertThat(ts_node_type(child), Equals("false")); child = ts_node_first_named_child_for_byte(root_node, false_index + 1); - AssertThat(ts_node_type(child, document), Equals("false")); + AssertThat(ts_node_type(child), Equals("false")); child = ts_node_first_named_child_for_byte(root_node, false_end_index); - AssertThat(ts_node_type(child, document), Equals("object")); + AssertThat(ts_node_type(child), Equals("object")); child = ts_node_first_named_child_for_byte(root_node, object_index); - AssertThat(ts_node_type(child, document), Equals("object")); + AssertThat(ts_node_type(child), Equals("object")); child = ts_node_first_named_child_for_byte(root_node, object_index + 1); - AssertThat(ts_node_type(child, document), Equals("object")); + AssertThat(ts_node_type(child), Equals("object")); child = ts_node_first_named_child_for_byte(root_node, object_end_index); AssertThat(child.subtree, Equals(nullptr)); }); @@ -255,14 +254,14 @@ describe("Node", [&]() { TSNode child6 = ts_node_child(root_node, 5); TSNode child7 = ts_node_child(root_node, 6); - AssertThat(ts_node_type(root_node, document), Equals("array")); - AssertThat(ts_node_type(child1, document), Equals("[")); - AssertThat(ts_node_type(child2, document), Equals("number")); - AssertThat(ts_node_type(child3, document), Equals(",")); - AssertThat(ts_node_type(child4, document), Equals("false")); - AssertThat(ts_node_type(child5, document), Equals(",")); - AssertThat(ts_node_type(child6, document), Equals("object")); - AssertThat(ts_node_type(child7, document), Equals("]")); + AssertThat(ts_node_type(root_node), Equals("array")); + AssertThat(ts_node_type(child1), Equals("[")); + AssertThat(ts_node_type(child2), Equals("number")); + AssertThat(ts_node_type(child3), Equals(",")); + AssertThat(ts_node_type(child4), Equals("false")); + AssertThat(ts_node_type(child5), Equals(",")); + AssertThat(ts_node_type(child6), Equals("object")); + AssertThat(ts_node_type(child7), Equals("]")); AssertThat(ts_node_is_named(root_node), IsTrue()); AssertThat(ts_node_is_named(child1), IsFalse()); @@ -303,13 +302,13 @@ describe("Node", [&]() { TSNode grandchild3 = ts_node_child(pair, 1); TSNode grandchild4 = ts_node_child(pair, 2); - AssertThat(ts_node_type(left_brace, document), Equals("{")); - AssertThat(ts_node_type(pair, document), Equals("pair")); - AssertThat(ts_node_type(right_brace, document), Equals("}")); + AssertThat(ts_node_type(left_brace), Equals("{")); + AssertThat(ts_node_type(pair), Equals("pair")); + AssertThat(ts_node_type(right_brace), Equals("}")); - AssertThat(ts_node_type(grandchild2, document), Equals("string")); - AssertThat(ts_node_type(grandchild3, document), Equals(":")); - AssertThat(ts_node_type(grandchild4, document), Equals("null")); + AssertThat(ts_node_type(grandchild2), Equals("string")); + AssertThat(ts_node_type(grandchild3), Equals(":")); + AssertThat(ts_node_type(grandchild4), Equals("null")); AssertThat(ts_node_parent(grandchild2), Equals(pair)); AssertThat(ts_node_parent(grandchild3), Equals(pair)); @@ -411,14 +410,14 @@ describe("Node", [&]() { describe("when there is a leaf node that spans the given range exactly", [&]() { it("returns that leaf node", [&]() { TSNode leaf = ts_node_named_descendant_for_byte_range(root_node, string_index, string_end_index - 1); - AssertThat(ts_node_type(leaf, document), Equals("string")); + AssertThat(ts_node_type(leaf), Equals("string")); AssertThat(ts_node_start_byte(leaf), Equals(string_index)); AssertThat(ts_node_end_byte(leaf), Equals(string_end_index)); AssertThat(ts_node_start_point(leaf), Equals({ 6, 4 })); AssertThat(ts_node_end_point(leaf), Equals({ 6, 7 })); leaf = ts_node_named_descendant_for_byte_range(root_node, number_index, number_end_index - 1); - AssertThat(ts_node_type(leaf, document), Equals("number")); + AssertThat(ts_node_type(leaf), Equals("number")); AssertThat(ts_node_start_byte(leaf), Equals(number_index)); AssertThat(ts_node_end_byte(leaf), Equals(number_end_index)); AssertThat(ts_node_start_point(leaf), Equals({ 3, 2 })); @@ -429,14 +428,14 @@ describe("Node", [&]() { describe("when there is a leaf node that extends beyond the given range", [&]() { it("returns that leaf node", [&]() { TSNode leaf = ts_node_named_descendant_for_byte_range(root_node, string_index, string_index + 1); - AssertThat(ts_node_type(leaf, document), Equals("string")); + AssertThat(ts_node_type(leaf), Equals("string")); AssertThat(ts_node_start_byte(leaf), Equals(string_index)); AssertThat(ts_node_end_byte(leaf), Equals(string_end_index)); AssertThat(ts_node_start_point(leaf), Equals({ 6, 4 })); AssertThat(ts_node_end_point(leaf), Equals({ 6, 7 })); leaf = ts_node_named_descendant_for_byte_range(root_node, string_index + 1, string_index + 2); - AssertThat(ts_node_type(leaf, document), Equals("string")); + AssertThat(ts_node_type(leaf), Equals("string")); AssertThat(ts_node_start_byte(leaf), Equals(string_index)); AssertThat(ts_node_end_byte(leaf), Equals(string_end_index)); AssertThat(ts_node_start_point(leaf), Equals({ 6, 4 })); @@ -447,7 +446,7 @@ describe("Node", [&]() { describe("when there is no leaf node that spans the given range", [&]() { it("returns the smallest node that does span the range", [&]() { TSNode pair_node = ts_node_named_descendant_for_byte_range(root_node, string_index, string_index + 3); - AssertThat(ts_node_type(pair_node, document), Equals("pair")); + AssertThat(ts_node_type(pair_node), Equals("pair")); AssertThat(ts_node_start_byte(pair_node), Equals(string_index)); AssertThat(ts_node_end_byte(pair_node), Equals(null_end_index)); AssertThat(ts_node_start_point(pair_node), Equals({ 6, 4 })); @@ -456,7 +455,7 @@ describe("Node", [&]() { it("does not return invisible nodes (repeats)", [&]() { TSNode node = ts_node_named_descendant_for_byte_range(root_node, number_end_index, number_end_index + 1); - AssertThat(ts_node_type(node, document), Equals("array")); + AssertThat(ts_node_type(node), Equals("array")); AssertThat(ts_node_start_byte(node), Equals(array_index)); AssertThat(ts_node_end_byte(node), Equals(array_end_index)); AssertThat(ts_node_start_point(node), Equals({ 2, 0 })); @@ -468,14 +467,14 @@ describe("Node", [&]() { describe("descendant_for_byte_range(start, end)", [&]() { it("returns the smallest node that spans the given byte offsets", [&]() { TSNode node1 = ts_node_descendant_for_byte_range(root_node, colon_index, colon_index); - AssertThat(ts_node_type(node1, document), Equals(":")); + AssertThat(ts_node_type(node1), Equals(":")); AssertThat(ts_node_start_byte(node1), Equals(colon_index)); AssertThat(ts_node_end_byte(node1), Equals(colon_index + 1)); AssertThat(ts_node_start_point(node1), Equals({ 6, 7 })); AssertThat(ts_node_end_point(node1), Equals({ 6, 8 })); TSNode node2 = ts_node_descendant_for_byte_range(root_node, string_index + 2, string_index + 4); - AssertThat(ts_node_type(node2, document), Equals("pair")); + AssertThat(ts_node_type(node2), Equals("pair")); AssertThat(ts_node_start_byte(node2), Equals(string_index)); AssertThat(ts_node_end_byte(node2), Equals(null_end_index)); AssertThat(ts_node_start_point(node2), Equals({ 6, 4 })); @@ -490,10 +489,10 @@ describe("Node", [&]() { uint32_t comma_position = input_string.find(","); TSNode node1 = ts_node_descendant_for_byte_range(root_node, comma_position, comma_position); - AssertThat(ts_node_type(node1, document), Equals(",")); + AssertThat(ts_node_type(node1), Equals(",")); TSNode node2 = ts_node_descendant_for_byte_range(root_node, 6, 10); - AssertThat(ts_node_type(node2, document), Equals("string")); + AssertThat(ts_node_type(node2), Equals("string")); AssertThat(ts_node_start_byte(node2), Equals(1)); AssertThat(ts_node_end_byte(node2), Equals(11)); }); @@ -502,14 +501,14 @@ describe("Node", [&]() { describe("descendant_for_point_range(start, end)", [&]() { it("returns the smallest concrete node that spans the given range", [&]() { TSNode node1 = ts_node_descendant_for_point_range(root_node, {6, 7}, {6, 7}); - AssertThat(ts_node_type(node1, document), Equals(":")); + AssertThat(ts_node_type(node1), Equals(":")); AssertThat(ts_node_start_byte(node1), Equals(colon_index)); AssertThat(ts_node_end_byte(node1), Equals(colon_index + 1)); AssertThat(ts_node_start_point(node1), Equals({ 6, 7 })); AssertThat(ts_node_end_point(node1), Equals({ 6, 8 })); TSNode node2 = ts_node_descendant_for_point_range(root_node, {6, 6}, {6, 8}); - AssertThat(ts_node_type(node2, document), Equals("pair")); + AssertThat(ts_node_type(node2), Equals("pair")); AssertThat(ts_node_start_byte(node2), Equals(string_index)); AssertThat(ts_node_end_byte(node2), Equals(null_end_index)); AssertThat(ts_node_start_point(node2), Equals({ 6, 4 })); diff --git a/test/runtime/parser_test.cc b/test/runtime/parser_test.cc index 204eedbd..2cf70b69 100644 --- a/test/runtime/parser_test.cc +++ b/test/runtime/parser_test.cc @@ -71,7 +71,7 @@ describe("Parser", [&]() { auto assert_root_node = [&](const string &expected) { TSNode node = ts_document_root_node(document); - char *node_string = ts_node_string(node, document); + char *node_string = ts_node_string(node); string actual(node_string); ts_free(node_string); AssertThat(actual, Equals(expected)); @@ -93,7 +93,7 @@ describe("Parser", [&]() { "(value (array (number) (ERROR (UNEXPECTED '@')) (true)))"); TSNode error = ts_node_named_child(ts_node_child(root, 0), 1); - AssertThat(ts_node_type(error, document), Equals("ERROR")); + AssertThat(ts_node_type(error), Equals("ERROR")); AssertThat(get_node_text(error), Equals("@@@@@,")); AssertThat(ts_node_child_count(error), Equals(2)); @@ -104,7 +104,7 @@ describe("Parser", [&]() { AssertThat(get_node_text(comma), Equals(",")); TSNode node_after_error = ts_node_next_named_sibling(error); - AssertThat(ts_node_type(node_after_error, document), Equals("true")); + AssertThat(ts_node_type(node_after_error), Equals("true")); AssertThat(get_node_text(node_after_error), Equals("true")); }); }); @@ -118,20 +118,20 @@ describe("Parser", [&]() { "(value (array (number) (ERROR (UNEXPECTED 'a')) (true)))"); TSNode error = ts_node_named_child(ts_node_child(root, 0), 1); - AssertThat(ts_node_type(error, document), Equals("ERROR")); + AssertThat(ts_node_type(error), Equals("ERROR")); AssertThat(get_node_text(error), Equals("faaaaalse,")); AssertThat(ts_node_child_count(error), Equals(2)); TSNode garbage = ts_node_child(error, 0); - AssertThat(ts_node_type(garbage, document), Equals("ERROR")); + AssertThat(ts_node_type(garbage), Equals("ERROR")); AssertThat(get_node_text(garbage), Equals("faaaaalse")); TSNode comma = ts_node_child(error, 1); - AssertThat(ts_node_type(comma, document), Equals(",")); + AssertThat(ts_node_type(comma), Equals(",")); AssertThat(get_node_text(comma), Equals(",")); TSNode last = ts_node_next_named_sibling(error); - AssertThat(ts_node_type(last, document), Equals("true")); + AssertThat(ts_node_type(last), Equals("true")); AssertThat(ts_node_start_byte(last), Equals(strlen(" [123, faaaaalse, "))); }); }); @@ -145,12 +145,12 @@ describe("Parser", [&]() { "(value (array (number) (true) (ERROR (false)) (true)))"); TSNode error = ts_node_named_child(ts_node_child(root, 0), 2); - AssertThat(ts_node_type(error, document), Equals("ERROR")); + AssertThat(ts_node_type(error), Equals("ERROR")); AssertThat(get_node_text(error), Equals("false")); AssertThat(ts_node_child_count(error), Equals(1)); TSNode last = ts_node_next_named_sibling(error); - AssertThat(ts_node_type(last, document), Equals("true")); + AssertThat(ts_node_type(last), Equals("true")); AssertThat(get_node_text(last), Equals("true")); }); }); @@ -186,7 +186,7 @@ describe("Parser", [&]() { ); TSNode error = ts_node_named_child(root, 1); - AssertThat(ts_node_type(error, document), Equals("ERROR")); + AssertThat(ts_node_type(error), Equals("ERROR")); AssertThat(ts_node_start_point(error), Equals({2, 0})); AssertThat(ts_node_end_point(error), Equals({2, 2})); }); @@ -303,7 +303,7 @@ describe("Parser", [&]() { "(program (expression_statement (binary_expression (identifier) (number))))"); TSNode node = ts_node_named_descendant_for_byte_range(root, 1, 1); - AssertThat(ts_node_type(node, document), Equals("identifier")); + AssertThat(ts_node_type(node), Equals("identifier")); AssertThat(ts_node_end_byte(node), Equals(strlen("abXYZc"))); }); }); @@ -322,7 +322,7 @@ describe("Parser", [&]() { "(program (expression_statement (binary_expression (identifier) (number))))"); TSNode node = ts_node_named_descendant_for_byte_range(root, 1, 1); - AssertThat(ts_node_type(node, document), Equals("identifier")); + AssertThat(ts_node_type(node), Equals("identifier")); AssertThat(ts_node_end_byte(node), Equals(strlen("abcXYZ"))); }); }); From 666dfb76d21166e315ddbf34049980e5081832b9 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 16:38:56 -0700 Subject: [PATCH 10/31] Remove document parameter from ts_node_type, ts_node_string Co-Authored-By: Rick Winfrey --- include/tree_sitter/runtime.h | 22 +++-- project.gyp | 1 + src/runtime/document.c | 15 ++-- src/runtime/document.h | 6 +- src/runtime/get_changed_ranges.c | 55 ++++++------- src/runtime/get_changed_ranges.h | 11 +-- src/runtime/tree_cursor.c | 134 +++++++++++++++++++++++++++++++ src/runtime/tree_cursor.h | 20 +++++ test/runtime/node_test.cc | 128 +++++++++++++++++++++++++++++ 9 files changed, 339 insertions(+), 53 deletions(-) create mode 100644 src/runtime/tree_cursor.c create mode 100644 src/runtime/tree_cursor.h diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index 934d5320..d3e282fb 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -14,6 +14,7 @@ extern "C" { typedef unsigned short TSSymbol; typedef struct TSLanguage TSLanguage; typedef struct TSDocument TSDocument; +typedef struct TSTreeCursor TSTreeCursor; typedef enum { TSInputEncodingUTF8, @@ -70,6 +71,12 @@ typedef struct { TSSymbol alias_symbol; } TSNode; +typedef struct { + TSRange **changed_ranges; + uint32_t *changed_range_count; + bool halt_on_error; +} TSParseOptions; + uint32_t ts_node_start_byte(TSNode); TSPoint ts_node_start_point(TSNode); uint32_t ts_node_end_byte(TSNode); @@ -112,19 +119,18 @@ void ts_document_print_debugging_graphs(TSDocument *, bool); void ts_document_edit(TSDocument *, TSInputEdit); void ts_document_parse(TSDocument *); void ts_document_parse_and_get_changed_ranges(TSDocument *, TSRange **, uint32_t *); - -typedef struct { - TSRange **changed_ranges; - uint32_t *changed_range_count; - bool halt_on_error; -} TSParseOptions; - void ts_document_parse_with_options(TSDocument *, TSParseOptions); - void ts_document_invalidate(TSDocument *); TSNode ts_document_root_node(const TSDocument *); +TSTreeCursor *ts_document_tree_cursor(const TSDocument *); uint32_t ts_document_parse_count(const TSDocument *); +void ts_tree_cursor_delete(TSTreeCursor *); +bool ts_tree_cursor_goto_first_child(TSTreeCursor *); +bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *); +bool ts_tree_cursor_goto_parent(TSTreeCursor *); +TSNode ts_tree_cursor_current_node(TSTreeCursor *); + uint32_t ts_language_symbol_count(const TSLanguage *); const char *ts_language_symbol_name(const TSLanguage *, TSSymbol); TSSymbolType ts_language_symbol_type(const TSLanguage *, TSSymbol); diff --git a/project.gyp b/project.gyp index b968a171..f59c3c91 100644 --- a/project.gyp +++ b/project.gyp @@ -96,6 +96,7 @@ 'src/runtime/parser.c', 'src/runtime/string_input.c', 'src/runtime/tree.c', + 'src/runtime/tree_cursor.c', 'src/runtime/utf16.c', 'externals/utf8proc/utf8proc.c', ], diff --git a/src/runtime/document.c b/src/runtime/document.c index ddd89b43..ab10ae49 100644 --- a/src/runtime/document.c +++ b/src/runtime/document.c @@ -3,6 +3,7 @@ #include "runtime/parser.h" #include "runtime/string_input.h" #include "runtime/document.h" +#include "runtime/tree_cursor.h" #include "runtime/get_changed_ranges.h" #define LOG(...) \ @@ -12,15 +13,15 @@ TSDocument *ts_document_new() { TSDocument *self = ts_calloc(1, sizeof(TSDocument)); parser_init(&self->parser); - array_init(&self->tree_path1); - array_init(&self->tree_path2); + array_init(&self->cursor1.stack); + array_init(&self->cursor2.stack); return self; } void ts_document_free(TSDocument *self) { if (self->tree) ts_tree_release(&self->parser.tree_pool, self->tree); - if (self->tree_path1.contents) array_delete(&self->tree_path1); - if (self->tree_path2.contents) array_delete(&self->tree_path2); + if (self->cursor1.stack.contents) array_delete(&self->cursor1.stack); + if (self->cursor2.stack.contents) array_delete(&self->cursor2.stack); parser_destroy(&self->parser); ts_document_set_input(self, (TSInput){ NULL, @@ -141,7 +142,7 @@ void ts_document_parse_with_options(TSDocument *self, TSParseOptions options) { if (options.changed_ranges && options.changed_range_count) { *options.changed_range_count = ts_tree_get_changed_ranges( - old_tree, tree, &self->tree_path1, &self->tree_path2, + old_tree, tree, &self->cursor1, &self->cursor2, self->parser.language, options.changed_ranges ); @@ -181,3 +182,7 @@ TSNode ts_document_root_node(const TSDocument *self) { uint32_t ts_document_parse_count(const TSDocument *self) { return self->parse_count; } + +TSTreeCursor *ts_document_tree_cursor(const TSDocument *self) { + return ts_tree_cursor_new(self); +} diff --git a/src/runtime/document.h b/src/runtime/document.h index 0be05f14..986fcec9 100644 --- a/src/runtime/document.h +++ b/src/runtime/document.h @@ -3,15 +3,15 @@ #include "runtime/parser.h" #include "runtime/tree.h" -#include "runtime/get_changed_ranges.h" +#include "runtime/tree_cursor.h" #include struct TSDocument { Parser parser; TSInput input; Tree *tree; - TreePath tree_path1; - TreePath tree_path2; + TSTreeCursor cursor1; + TSTreeCursor cursor2; size_t parse_count; bool valid; bool owns_input; diff --git a/src/runtime/get_changed_ranges.c b/src/runtime/get_changed_ranges.c index 26211613..7632af24 100644 --- a/src/runtime/get_changed_ranges.c +++ b/src/runtime/get_changed_ranges.c @@ -2,6 +2,7 @@ #include "runtime/tree.h" #include "runtime/language.h" #include "runtime/error_costs.h" +#include "runtime/tree_cursor.h" #include // #define DEBUG_GET_CHANGED_RANGES @@ -24,22 +25,22 @@ static void range_array_add(RangeArray *results, TSPoint start, TSPoint end) { } typedef struct { - TreePath path; + TSTreeCursor cursor; const TSLanguage *language; unsigned visible_depth; bool in_padding; } Iterator; -static Iterator iterator_new(TreePath *path, Tree *tree, const TSLanguage *language) { - array_clear(path); - array_push(path, ((TreePathEntry){ +static Iterator iterator_new(TSTreeCursor *cursor, Tree *tree, const TSLanguage *language) { + array_clear(&cursor->stack); + array_push(&cursor->stack, ((TreeCursorEntry){ .tree = tree, .position = length_zero(), .child_index = 0, .structural_child_index = 0, })); return (Iterator) { - .path = *path, + .cursor = *cursor, .language = language, .visible_depth = 1, .in_padding = false, @@ -47,11 +48,11 @@ static Iterator iterator_new(TreePath *path, Tree *tree, const TSLanguage *langu } static bool iterator_done(Iterator *self) { - return self->path.size == 0; + return self->cursor.stack.size == 0; } Length iterator_start_position(Iterator *self) { - TreePathEntry entry = *array_back(&self->path); + TreeCursorEntry entry = *array_back(&self->cursor.stack); if (self->in_padding) { return entry.position; } else { @@ -60,7 +61,7 @@ Length iterator_start_position(Iterator *self) { } Length iterator_end_position(Iterator *self) { - TreePathEntry entry = *array_back(&self->path); + TreeCursorEntry entry = *array_back(&self->cursor.stack); Length result = length_add(entry.position, entry.tree->padding); if (self->in_padding) { return result; @@ -70,10 +71,10 @@ Length iterator_end_position(Iterator *self) { } static bool iterator_tree_is_visible(const Iterator *self) { - TreePathEntry entry = *array_back(&self->path); + TreeCursorEntry entry = *array_back(&self->cursor.stack); if (entry.tree->visible) return true; - if (self->path.size > 1) { - Tree *parent = self->path.contents[self->path.size - 2].tree; + if (self->cursor.stack.size > 1) { + Tree *parent = self->cursor.stack.contents[self->cursor.stack.size - 2].tree; const TSSymbol *alias_sequence = ts_language_alias_sequence(self->language, parent->alias_sequence_id); return alias_sequence && alias_sequence[entry.structural_child_index] != 0; } @@ -82,7 +83,7 @@ static bool iterator_tree_is_visible(const Iterator *self) { static void iterator_get_visible_state(const Iterator *self, Tree **tree, TSSymbol *alias_symbol, uint32_t *start_byte) { - uint32_t i = self->path.size - 1; + uint32_t i = self->cursor.stack.size - 1; if (self->in_padding) { if (i == 0) return; @@ -90,10 +91,10 @@ static void iterator_get_visible_state(const Iterator *self, Tree **tree, } for (; i + 1 > 0; i--) { - TreePathEntry entry = self->path.contents[i]; + TreeCursorEntry entry = self->cursor.stack.contents[i]; if (i > 0) { - Tree *parent = self->path.contents[i - 1].tree; + Tree *parent = self->cursor.stack.contents[i - 1].tree; const TSSymbol *alias_sequence = ts_language_alias_sequence( self->language, parent->alias_sequence_id @@ -114,8 +115,8 @@ static void iterator_get_visible_state(const Iterator *self, Tree **tree, static void iterator_ascend(Iterator *self) { if (iterator_done(self)) return; if (iterator_tree_is_visible(self) && !self->in_padding) self->visible_depth--; - if (array_back(&self->path)->child_index > 0) self->in_padding = false; - self->path.size--; + if (array_back(&self->cursor.stack)->child_index > 0) self->in_padding = false; + self->cursor.stack.size--; } static bool iterator_descend(Iterator *self, uint32_t goal_position) { @@ -124,7 +125,7 @@ static bool iterator_descend(Iterator *self, uint32_t goal_position) { bool did_descend; do { did_descend = false; - TreePathEntry entry = *array_back(&self->path); + TreeCursorEntry entry = *array_back(&self->cursor.stack); Length position = entry.position; uint32_t structural_child_index = 0; for (uint32_t i = 0; i < entry.tree->children.size; i++) { @@ -133,7 +134,7 @@ static bool iterator_descend(Iterator *self, uint32_t goal_position) { Length child_right = length_add(child_left, child->size); if (child_right.bytes > goal_position) { - array_push(&self->path, ((TreePathEntry){ + array_push(&self->cursor.stack, ((TreeCursorEntry){ .tree = child, .position = position, .child_index = i, @@ -174,10 +175,10 @@ static void iterator_advance(Iterator *self) { for (;;) { if (iterator_tree_is_visible(self)) self->visible_depth--; - TreePathEntry entry = array_pop(&self->path); + TreeCursorEntry entry = array_pop(&self->cursor.stack); if (iterator_done(self)) return; - Tree *parent = array_back(&self->path)->tree; + Tree *parent = array_back(&self->cursor.stack)->tree; uint32_t child_index = entry.child_index + 1; if (parent->children.size > child_index) { Length position = length_add(entry.position, ts_tree_total_size(entry.tree)); @@ -185,7 +186,7 @@ static void iterator_advance(Iterator *self) { if (!entry.tree->extra) structural_child_index++; Tree *next_child = parent->children.contents[child_index]; - array_push(&self->path, ((TreePathEntry){ + array_push(&self->cursor.stack, ((TreeCursorEntry){ .tree = next_child, .position = position, .child_index = child_index, @@ -246,7 +247,7 @@ IteratorComparison iterator_compare(const Iterator *old_iter, const Iterator *ne #ifdef DEBUG_GET_CHANGED_RANGES static inline void iterator_print_state(Iterator *self) { - TreePathEntry entry = *array_back(&self->path); + TreeCursorEntry entry = *array_back(&self->cursor.stack); TSPoint start = iterator_start_position(self).extent; TSPoint end = iterator_end_position(self).extent; const char *name = ts_language_symbol_name(self->language, entry.tree->symbol); @@ -261,12 +262,12 @@ static inline void iterator_print_state(Iterator *self) { #endif unsigned ts_tree_get_changed_ranges(Tree *old_tree, Tree *new_tree, - TreePath *path1, TreePath *path2, + TSTreeCursor *cursor1, TSTreeCursor *cursor2, const TSLanguage *language, TSRange **ranges) { RangeArray results = array_new(); - Iterator old_iter = iterator_new(path1, old_tree, language); - Iterator new_iter = iterator_new(path2, new_tree, language); + Iterator old_iter = iterator_new(cursor1, old_tree, language); + Iterator new_iter = iterator_new(cursor2, new_tree, language); Length position = iterator_start_position(&old_iter); Length next_position = iterator_start_position(&new_iter); @@ -348,8 +349,8 @@ unsigned ts_tree_get_changed_ranges(Tree *old_tree, Tree *new_tree, position = next_position; } while (!iterator_done(&old_iter) && !iterator_done(&new_iter)); - *path1 = old_iter.path; - *path2 = new_iter.path; + *cursor1 = old_iter.cursor; + *cursor2 = new_iter.cursor; *ranges = results.contents; return results.size; } diff --git a/src/runtime/get_changed_ranges.h b/src/runtime/get_changed_ranges.h index 360cdbd4..fa408aaa 100644 --- a/src/runtime/get_changed_ranges.h +++ b/src/runtime/get_changed_ranges.h @@ -3,17 +3,8 @@ #include "runtime/tree.h" -typedef struct { - Tree *tree; - Length position; - uint32_t child_index; - uint32_t structural_child_index; -} TreePathEntry; - -typedef Array(TreePathEntry) TreePath; - unsigned ts_tree_get_changed_ranges( - Tree *old_tree, Tree *new_tree, TreePath *path1, TreePath *path2, + Tree *old_tree, Tree *new_tree, TSTreeCursor *cursor1, TSTreeCursor *cursor2, const TSLanguage *language, TSRange **ranges ); diff --git a/src/runtime/tree_cursor.c b/src/runtime/tree_cursor.c new file mode 100644 index 00000000..db8298d2 --- /dev/null +++ b/src/runtime/tree_cursor.c @@ -0,0 +1,134 @@ +#include "tree_sitter/runtime.h" +#include "runtime/alloc.h" +#include "runtime/tree_cursor.h" +#include "runtime/document.h" +#include "runtime/language.h" + +TSTreeCursor *ts_tree_cursor_new(const TSDocument *document) { + TSTreeCursor *self = ts_malloc(sizeof(TSTreeCursor)); + self->document = document; + array_init(&self->stack); + array_push(&self->stack, ((TreeCursorEntry) { + .tree = document->tree, + .position = length_zero(), + .child_index = 0, + .structural_child_index = 0, + })); + return self; +} + +void ts_tree_cursor_delete(TSTreeCursor *self) { + array_delete(&self->stack); + ts_free(self); +} + +bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { + TreeCursorEntry *last_entry = array_back(&self->stack); + Tree *tree = last_entry->tree; + Length position = last_entry->position; + + bool did_descend; + do { + did_descend = false; + + uint32_t structural_child_index = 0; + for (uint32_t i = 0; i < tree->children.size; i++) { + Tree *child = tree->children.contents[i]; + if (child->visible || child->visible_child_count > 0) { + array_push(&self->stack, ((TreeCursorEntry) { + .tree = child, + .child_index = i, + .structural_child_index = structural_child_index, + .position = position, + })); + + if (child->visible) { + return true; + } else { + tree = child; + did_descend = true; + break; + } + } + if (!child->extra) structural_child_index++; + position = length_add(position, ts_tree_total_size(child)); + } + } while (did_descend); + + return false; +} + +bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { + TreeCursorEntry *child_entry = array_back(&self->stack); + + for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) { + TreeCursorEntry *parent_entry = &self->stack.contents[i]; + + Tree *parent = parent_entry->tree; + uint32_t child_index = child_entry->child_index; + uint32_t structural_child_index = child_entry->structural_child_index; + Length position = child_entry->position; + Tree *child = parent->children.contents[child_index]; + + while (++child_index < parent->children.size) { + if (!child->extra) structural_child_index++; + position = length_add(position, ts_tree_total_size(child)); + child = parent->children.contents[child_index]; + + if (child->visible || child->visible_child_count > 0) { + self->stack.contents[i + 1] = (TreeCursorEntry) { + .tree = child, + .child_index = child_index, + .structural_child_index = structural_child_index, + .position = position, + }; + self->stack.size = i + 2; + + if (child->visible) { + return true; + } else { + ts_tree_cursor_goto_first_child(self); + return true; + } + } + } + + child_entry = parent_entry; + if (parent->visible) break; + } + + return false; +} + +bool ts_tree_cursor_goto_parent(TSTreeCursor *self) { + for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) { + TreeCursorEntry *entry = &self->stack.contents[i]; + if (entry->tree->visible) { + self->stack.size = i + 1; + return true; + } + } + return false; +} + +TSNode ts_tree_cursor_current_node(TSTreeCursor *self) { + TreeCursorEntry *last_entry = array_back(&self->stack); + TSSymbol alias_symbol = 0; + if (self->stack.size > 1) { + TreeCursorEntry *parent_entry = &self->stack.contents[self->stack.size - 2]; + const TSSymbol *alias_sequence = ts_language_alias_sequence( + self->document->parser.language, + parent_entry->tree->alias_sequence_id + ); + if (alias_sequence) { + alias_symbol = alias_sequence[last_entry->structural_child_index]; + } + } + return (TSNode) { + .document = self->document, + .subtree = last_entry->tree, + .position = last_entry->position.extent, + .byte = last_entry->position.bytes, + .alias_symbol = alias_symbol, + }; +} diff --git a/src/runtime/tree_cursor.h b/src/runtime/tree_cursor.h new file mode 100644 index 00000000..33e1d802 --- /dev/null +++ b/src/runtime/tree_cursor.h @@ -0,0 +1,20 @@ +#ifndef RUNTIME_TREE_CURSOR_H_ +#define RUNTIME_TREE_CURSOR_H_ + +#include "runtime/tree.h" + +typedef struct { + Tree *tree; + Length position; + uint32_t child_index; + uint32_t structural_child_index; +} TreeCursorEntry; + +struct TSTreeCursor { + const TSDocument *document; + Array(TreeCursorEntry) stack; +}; + +TSTreeCursor *ts_tree_cursor_new(const TSDocument *); + +#endif // RUNTIME_TREE_CURSOR_H_ diff --git a/test/runtime/node_test.cc b/test/runtime/node_test.cc index 91717ed2..808084be 100644 --- a/test/runtime/node_test.cc +++ b/test/runtime/node_test.cc @@ -517,4 +517,132 @@ describe("Node", [&]() { }); }); +describe("TreeCursor", [&]() { + TSDocument *document; + TSTreeCursor *cursor; + + before_each([&]() { + record_alloc::start(); + + document = ts_document_new(); + ts_document_set_language(document, load_real_language("json")); + ts_document_set_input_string(document, json_string.c_str()); + ts_document_parse(document); + + cursor = ts_document_tree_cursor(document); + }); + + after_each([&]() { + ts_tree_cursor_delete(cursor); + ts_document_free(document); + + record_alloc::stop(); + AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); + }); + + it("can walk the tree", [&]() { + TSNode node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("value")); + AssertThat(ts_node_start_byte(node), Equals(array_index)); + + AssertThat(ts_tree_cursor_goto_first_child(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("array")); + AssertThat(ts_node_start_byte(node), Equals(array_index)); + + AssertThat(ts_tree_cursor_goto_first_child(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("[")); + AssertThat(ts_node_start_byte(node), Equals(array_index)); + + // Cannot descend into a node with no children + AssertThat(ts_tree_cursor_goto_first_child(cursor), IsFalse()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("[")); + AssertThat(ts_node_start_byte(node), Equals(array_index)); + + AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("number")); + AssertThat(ts_node_start_byte(node), Equals(number_index)); + + AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals(",")); + AssertThat(ts_node_start_byte(node), Equals(number_end_index)); + + AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("false")); + AssertThat(ts_node_start_byte(node), Equals(false_index)); + + AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals(",")); + AssertThat(ts_node_start_byte(node), Equals(false_end_index)); + + AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("object")); + AssertThat(ts_node_start_byte(node), Equals(object_index)); + + AssertThat(ts_tree_cursor_goto_first_child(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("{")); + AssertThat(ts_node_start_byte(node), Equals(object_index)); + + AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("pair")); + AssertThat(ts_node_start_byte(node), Equals(string_index)); + + AssertThat(ts_tree_cursor_goto_first_child(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("string")); + AssertThat(ts_node_start_byte(node), Equals(string_index)); + + AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals(":")); + AssertThat(ts_node_start_byte(node), Equals(string_end_index)); + + AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("null")); + AssertThat(ts_node_start_byte(node), Equals(null_index)); + + // Cannot move beyond a node with no next sibling + AssertThat(ts_tree_cursor_goto_next_sibling(cursor), IsFalse()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("null")); + AssertThat(ts_node_start_byte(node), Equals(null_index)); + + AssertThat(ts_tree_cursor_goto_parent(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("pair")); + AssertThat(ts_node_start_byte(node), Equals(string_index)); + + AssertThat(ts_tree_cursor_goto_parent(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("object")); + AssertThat(ts_node_start_byte(node), Equals(object_index)); + + AssertThat(ts_tree_cursor_goto_parent(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("array")); + AssertThat(ts_node_start_byte(node), Equals(array_index)); + + AssertThat(ts_tree_cursor_goto_parent(cursor), IsTrue()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("value")); + AssertThat(ts_node_start_byte(node), Equals(array_index)); + + // The root node doesn't have a parent. + AssertThat(ts_tree_cursor_goto_parent(cursor), IsFalse()); + node = ts_tree_cursor_current_node(cursor); + AssertThat(ts_node_type(node), Equals("value")); + AssertThat(ts_node_start_byte(node), Equals(array_index)); + }); +}); + END_TEST From 78d158899ec0b14fa796430f8830834b037df7c4 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 17:30:13 -0700 Subject: [PATCH 11/31] Use atomic operations for updating syntax tree reference counts Co-Authored-By: Rick Winfrey --- src/runtime/atomic.h | 30 ++++++++++++++++++++++++++++++ src/runtime/tree.c | 6 +++--- src/runtime/tree.h | 2 +- 3 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 src/runtime/atomic.h diff --git a/src/runtime/atomic.h b/src/runtime/atomic.h new file mode 100644 index 00000000..d1ab1f23 --- /dev/null +++ b/src/runtime/atomic.h @@ -0,0 +1,30 @@ +#ifndef RUNTIME_ATOMIC_H_ +#define RUNTIME_ATOMIC_H_ + +#include + +#ifdef _WIN32 + +#include + +static inline uint32_t atomic_inc(volatile uint32_t *p) { + return InterlockedIncrement(p); +} + +static inline uint32_t atomic_dec(volatile uint32_t *p) { + return InterlockedDecrement(p); +} + +#else + +static inline uint32_t atomic_inc(volatile uint32_t *p) { + return __sync_add_and_fetch(p, 1u); +} + +static inline uint32_t atomic_dec(volatile uint32_t *p) { + return __sync_sub_and_fetch(p, 1u); +} + +#endif + +#endif // RUNTIME_ATOMIC_H_ diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 0174d3d8..85707c1b 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -5,6 +5,7 @@ #include #include #include "runtime/alloc.h" +#include "runtime/atomic.h" #include "runtime/tree.h" #include "runtime/length.h" #include "runtime/language.h" @@ -356,7 +357,7 @@ Tree *ts_tree_make_missing_leaf(TreePool *pool, TSSymbol symbol, const TSLanguag void ts_tree_retain(Tree *self) { assert(self->ref_count > 0); - self->ref_count++; + atomic_inc(&self->ref_count); assert(self->ref_count != 0); } @@ -366,8 +367,7 @@ void ts_tree_release(TreePool *pool, Tree *self) { while (pool->tree_stack.size > 0) { Tree *tree = array_pop(&pool->tree_stack); assert(tree->ref_count > 0); - tree->ref_count--; - if (tree->ref_count == 0) { + if (atomic_dec(&tree->ref_count) == 0) { if (tree->children.size > 0) { for (uint32_t i = 0; i < tree->children.size; i++) { array_push(&pool->tree_stack, tree->children.contents[i]); diff --git a/src/runtime/tree.h b/src/runtime/tree.h index da5f1e90..0eb6f929 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -29,7 +29,7 @@ typedef Array(Tree *) TreeArray; struct Tree { Length padding; Length size; - uint32_t ref_count; + volatile uint32_t ref_count; uint32_t bytes_scanned; uint32_t error_cost; uint32_t node_count; From a53d0b43a1cc721c419abbf6d85fb32a9b4a0612 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 9 May 2018 22:55:54 -0700 Subject: [PATCH 12/31] Remove unnecessary include --- test/helpers/tree_helpers.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/test/helpers/tree_helpers.cc b/test/helpers/tree_helpers.cc index 553f8b49..9a86b364 100644 --- a/test/helpers/tree_helpers.cc +++ b/test/helpers/tree_helpers.cc @@ -1,7 +1,6 @@ #include "test_helper.h" #include "helpers/tree_helpers.h" #include "helpers/point_helpers.h" -#include "runtime/document.h" #include using std::string; From 59694e60faebfbf8f19f0c31f7d751f02c5debf0 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 10 May 2018 09:13:46 -0700 Subject: [PATCH 13/31] Rename ts_tree_assign_parents -> ts_tree_balance --- src/runtime/parser.c | 2 +- src/runtime/tree.c | 3 ++- src/runtime/tree.h | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/runtime/parser.c b/src/runtime/parser.c index cac6c9e1..f1ba0d16 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -1336,7 +1336,7 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_err reusable_node_delete(&reusable_node); ts_stack_clear(self->stack); parser__set_cached_token(self, 0, NULL, NULL); - ts_tree_assign_parents(self->finished_tree, &self->tree_pool, self->language); + ts_tree_balance(self->finished_tree, &self->tree_pool, self->language); LOG("done"); LOG_TREE(); diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 85707c1b..e80a1f68 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -208,11 +208,12 @@ static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *lang } } -void ts_tree_assign_parents(Tree *self, TreePool *pool, const TSLanguage *language) { +void ts_tree_balance(Tree *self, TreePool *pool, const TSLanguage *language) { array_clear(&pool->tree_stack); array_push(&pool->tree_stack, self); while (pool->tree_stack.size > 0) { Tree *tree = array_pop(&pool->tree_stack); + assert(tree); if (tree->repeat_depth > 0) { if (tree->children.contents[0]->repeat_depth > tree->children.contents[1]->repeat_depth) { diff --git a/src/runtime/tree.h b/src/runtime/tree.h index 0eb6f929..ce96e41d 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -99,7 +99,7 @@ void ts_tree_release(TreePool *, Tree *tree); bool ts_tree_eq(const Tree *tree1, const Tree *tree2); int ts_tree_compare(const Tree *tree1, const Tree *tree2); void ts_tree_set_children(Tree *, TreeArray *, const TSLanguage *); -void ts_tree_assign_parents(Tree *, TreePool *, const TSLanguage *); +void ts_tree_balance(Tree *, TreePool *, const TSLanguage *); void ts_tree_edit(Tree *, const TSInputEdit *edit); char *ts_tree_string(const Tree *, const TSLanguage *, bool include_all); void ts_tree_print_dot_graph(const Tree *, const TSLanguage *, FILE *); From eff7283325d1fa9bfe0d394a894bd825fca73b8a Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 10 May 2018 09:48:50 -0700 Subject: [PATCH 14/31] Add assertion to satisfy clang static analyzer Co-Authored-By: Rick Winfrey --- src/runtime/tree.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/runtime/tree.c b/src/runtime/tree.c index e80a1f68..a7e9fb99 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -200,6 +200,7 @@ static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *lang while (stack->size > initial_stack_size) { tree = array_pop(stack); + assert(tree); Tree *child = tree->children.contents[0]; Tree *grandchild = child->children.contents[1]; ts_tree_set_children(grandchild, &grandchild->children, language); From df79ff59971465fc58f670ef40781a67aae21693 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 10 May 2018 12:04:18 -0700 Subject: [PATCH 15/31] Refactor ts_tree_edit Co-Authored-By: Rick Winfrey --- src/runtime/tree.c | 142 +++++++++++++++++--------------------- test/runtime/tree_test.cc | 4 -- 2 files changed, 65 insertions(+), 81 deletions(-) diff --git a/src/runtime/tree.c b/src/runtime/tree.c index a7e9fb99..13a2f08d 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -11,6 +11,12 @@ #include "runtime/language.h" #include "runtime/error_costs.h" +typedef struct { + Length start; + Length added; + Length removed; +} Edit; + TSStateId TS_TREE_STATE_NONE = USHRT_MAX; // ExternalTokenState @@ -432,12 +438,8 @@ int ts_tree_compare(const Tree *left, const Tree *right) { return 0; } -static inline long min_byte(long a, long b) { - return a <= b ? a : b; -} - -bool ts_tree_invalidate_lookahead(Tree *self, uint32_t edit_byte_offset) { - if (edit_byte_offset >= self->bytes_scanned) return false; +void ts_tree_invalidate_lookahead(Tree *self, uint32_t edit_byte_offset) { + if (edit_byte_offset >= self->bytes_scanned) return; self->has_changes = true; if (self->children.size > 0) { uint32_t child_start_byte = 0; @@ -448,95 +450,70 @@ bool ts_tree_invalidate_lookahead(Tree *self, uint32_t edit_byte_offset) { child_start_byte += ts_tree_total_bytes(child); } } - return true; } -static inline TSPoint ts_tree_total_extent(const Tree *self) { - return point_add(self->padding.extent, self->size.extent); -} - -void ts_tree_edit(Tree *self, const TSInputEdit *edit) { - uint32_t old_end_byte = edit->start_byte + edit->bytes_removed; - uint32_t new_end_byte = edit->start_byte + edit->bytes_added; - TSPoint old_end_point = point_add(edit->start_point, edit->extent_removed); - TSPoint new_end_point = point_add(edit->start_point, edit->extent_added); - - assert(old_end_byte <= ts_tree_total_bytes(self)); +void ts_tree__edit(Tree *self, Edit edit) { + Length new_end = length_add(edit.start, edit.added); + Length old_end = length_add(edit.start, edit.removed); self->has_changes = true; - if (edit->start_byte < self->padding.bytes) { - if (self->padding.bytes >= old_end_byte) { - uint32_t trailing_padding_bytes = self->padding.bytes - old_end_byte; - TSPoint trailing_padding_extent = point_sub(self->padding.extent, old_end_point); - self->padding.bytes = new_end_byte + trailing_padding_bytes; - self->padding.extent = point_add(new_end_point, trailing_padding_extent); - } else { - uint32_t removed_content_bytes = old_end_byte - self->padding.bytes; - TSPoint removed_content_extent = point_sub(old_end_point, self->padding.extent); - self->size.bytes = self->size.bytes - removed_content_bytes; - self->size.extent = point_sub(self->size.extent, removed_content_extent); - self->padding.bytes = new_end_byte; - self->padding.extent = new_end_point; - } - } else if (edit->start_byte == self->padding.bytes && edit->bytes_removed == 0) { - self->padding.bytes = self->padding.bytes + edit->bytes_added; - self->padding.extent = point_add(self->padding.extent, edit->extent_added); + if (old_end.bytes <= self->padding.bytes) { + self->padding = length_add(new_end, length_sub(self->padding, old_end)); + } else if (edit.start.bytes < self->padding.bytes) { + self->size = length_sub(self->size, length_sub(old_end, self->padding)); + self->padding = new_end; + } else if (edit.start.bytes == self->padding.bytes && edit.removed.bytes == 0) { + self->padding = length_add(self->padding, edit.added); } else { - uint32_t trailing_content_bytes = ts_tree_total_bytes(self) - old_end_byte; - TSPoint trailing_content_extent = point_sub(ts_tree_total_extent(self), old_end_point); - self->size.bytes = new_end_byte + trailing_content_bytes - self->padding.bytes; - self->size.extent = point_sub(point_add(new_end_point, trailing_content_extent), self->padding.extent); + Length new_total_size = length_add(new_end, length_sub(ts_tree_total_size(self), old_end)); + self->size = length_sub(new_total_size, self->padding); } - bool found_first_child = false; - long remaining_bytes_to_delete = 0; - TSPoint remaining_extent_to_delete = {0, 0}; Length child_left, child_right = length_zero(); for (uint32_t i = 0; i < self->children.size; i++) { Tree *child = self->children.contents[i]; + Length child_size = ts_tree_total_size(child); child_left = child_right; - child_right = length_add(child_left, ts_tree_total_size(child)); + child_right = length_add(child_left, child_size); - if (!found_first_child && child_right.bytes >= edit->start_byte) { - found_first_child = true; - TSInputEdit child_edit = { - .start_byte = edit->start_byte - child_left.bytes, - .bytes_added = edit->bytes_added, - .bytes_removed = edit->bytes_removed, - .start_point = point_sub(edit->start_point, child_left.extent), - .extent_added = edit->extent_added, - .extent_removed = edit->extent_removed, + if (child_left.bytes > old_end.bytes || + (child_left.bytes == old_end.bytes && child_size.bytes > 0 && i > 0)) break; + + if (child_right.bytes > edit.start.bytes || + (child_right.bytes == edit.start.bytes && edit.removed.bytes == 0)) { + Edit child_edit = { + .start = length_sub(edit.start, child_left), + .added = edit.added, + .removed = edit.removed, }; - if (old_end_byte > child_right.bytes) { - child_edit.bytes_removed = child_right.bytes - edit->start_byte; - child_edit.extent_removed = point_sub(child_right.extent, edit->start_point); - remaining_bytes_to_delete = old_end_byte - child_right.bytes; - remaining_extent_to_delete = point_sub(old_end_point, child_right.extent); + if (edit.start.bytes < child_left.bytes) { + child_edit.start = length_zero(); } - ts_tree_edit(child, &child_edit); - } else if (remaining_bytes_to_delete > 0) { - TSInputEdit child_edit = { - .start_byte = 0, - .bytes_added = 0, - .bytes_removed = min_byte(remaining_bytes_to_delete, ts_tree_total_bytes(child)), - .start_point = {0, 0}, - .extent_added = {0, 0}, - .extent_removed = point_min(remaining_extent_to_delete, ts_tree_total_size(child).extent), - }; - remaining_bytes_to_delete -= child_edit.bytes_removed; - remaining_extent_to_delete = point_sub(remaining_extent_to_delete, child_edit.extent_removed); - ts_tree_edit(child, &child_edit); - } else { - ts_tree_invalidate_lookahead(child, edit->start_byte - child_left.bytes); - } + if (old_end.bytes > child_right.bytes) { + child_edit.removed = length_sub(child_size, child_edit.start); + } - child_right = length_add(child_left, ts_tree_total_size(child)); + edit.added = length_zero(); + edit.removed = length_sub(edit.removed, child_edit.removed); + + ts_tree__edit(child, child_edit); + } else if (child_left.bytes <= edit.start.bytes) { + ts_tree_invalidate_lookahead(child, edit.start.bytes - child_left.bytes); + } } } +void ts_tree_edit(Tree *self, const TSInputEdit *edit) { + ts_tree__edit(self, (Edit) { + .start = {edit->start_byte, edit->start_point}, + .added = {edit->bytes_added, edit->extent_added}, + .removed = {edit->bytes_removed, edit->extent_removed}, + }); +} + Tree *ts_tree_last_external_token(Tree *tree) { if (!tree->has_external_tokens) return NULL; while (tree->children.size > 0) { @@ -649,9 +626,20 @@ void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset, if (self->extra) fprintf(f, ", fontcolor=gray"); - fprintf(f, ", tooltip=\"address:%p\nrange:%u - %u\nstate:%d\nerror-cost:%u\nrepeat-depth:%u\"]\n", - self, byte_offset, byte_offset + ts_tree_total_bytes(self), self->parse_state, - self->error_cost, self->repeat_depth); + fprintf(f, ", tooltip=\"" + "address:%p\n" + "range:%u - %u\n" + "state:%d\n" + "error-cost:%u\n" + "repeat-depth:%u\n" + "bytes-scanned:%u\"]\n", + self, + byte_offset, byte_offset + ts_tree_total_bytes(self), + self->parse_state, + self->error_cost, + self->repeat_depth, + self->bytes_scanned + ); const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); uint32_t structural_child_index = 0; diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc index 262572b1..e80743e7 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/tree_test.cc @@ -255,8 +255,6 @@ describe("Tree", []() { ts_tree_edit(tree, &edit); assert_consistent(tree); - assert_consistent(tree); - AssertThat(tree->has_changes, IsTrue()); AssertThat(tree->padding, Equals({4, {0, 4}})); AssertThat(tree->size, Equals({13, {0, 13}})); @@ -305,8 +303,6 @@ describe("Tree", []() { ts_tree_edit(tree, &edit); assert_consistent(tree); - assert_consistent(tree); - AssertThat(tree->has_changes, IsTrue()); AssertThat(tree->padding, Equals({4, {0, 4}})); AssertThat(tree->size, Equals({4, {0, 4}})); From 09e663c7d1118da4aaf5b8832e54e32bd0258dc2 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 10 May 2018 12:23:05 -0700 Subject: [PATCH 16/31] Make ts_tree_edit return a new tree rather than mutating its argument Co-Authored-By: Rick Winfrey --- src/runtime/document.c | 2 +- src/runtime/tree.c | 78 +++++++++++++++++++++++++-------------- src/runtime/tree.h | 2 +- test/runtime/tree_test.cc | 14 +++---- 4 files changed, 59 insertions(+), 37 deletions(-) diff --git a/src/runtime/document.c b/src/runtime/document.c index ab10ae49..9a13ca30 100644 --- a/src/runtime/document.c +++ b/src/runtime/document.c @@ -97,7 +97,7 @@ void ts_document_edit(TSDocument *self, TSInputEdit edit) { if (edit.bytes_removed > max_bytes - edit.start_byte) edit.bytes_removed = max_bytes - edit.start_byte; - ts_tree_edit(self->tree, &edit); + self->tree = ts_tree_edit(self->tree, &edit, &self->parser.tree_pool); if (self->parser.print_debugging_graphs) { ts_tree_print_dot_graph(self->tree, self->parser.language, stderr); diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 13a2f08d..602392f4 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -58,8 +58,9 @@ bool ts_tree_array_copy(TreeArray self, TreeArray *dest) { if (self.capacity > 0) { contents = ts_calloc(self.capacity, sizeof(Tree *)); memcpy(contents, self.contents, self.size * sizeof(Tree *)); - for (uint32_t i = 0; i < self.size; i++) + for (uint32_t i = 0; i < self.size; i++) { ts_tree_retain(contents[i]); + } } dest->size = self.size; @@ -172,10 +173,23 @@ Tree *ts_tree_make_error(TreePool *pool, Length size, Length padding, int32_t lo Tree *ts_tree_make_copy(TreePool *pool, Tree *self) { Tree *result = ts_tree_pool_allocate(pool); *result = *self; + if (result->children.size > 0) { + ts_tree_array_copy(self->children, &result->children); + } result->ref_count = 1; return result; } +static Tree *ts_tree_make_mut(TreePool *pool, Tree *self) { + if (self->ref_count == 1) { + return self; + } else { + Tree *result = ts_tree_make_copy(pool, self); + ts_tree_release(pool, self); + return result; + } +} + static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *language, TreeArray *stack) { unsigned initial_stack_size = stack->size; @@ -438,42 +452,48 @@ int ts_tree_compare(const Tree *left, const Tree *right) { return 0; } -void ts_tree_invalidate_lookahead(Tree *self, uint32_t edit_byte_offset) { - if (edit_byte_offset >= self->bytes_scanned) return; - self->has_changes = true; - if (self->children.size > 0) { +Tree *ts_tree_invalidate_lookahead(Tree *self, uint32_t edit_byte_offset, TreePool *pool) { + if (edit_byte_offset >= self->bytes_scanned) return self; + + Tree *result = ts_tree_make_mut(pool, self); + result->has_changes = true; + + if (result->children.size > 0) { uint32_t child_start_byte = 0; - for (uint32_t i = 0; i < self->children.size; i++) { - Tree *child = self->children.contents[i]; + for (uint32_t i = 0; i < result->children.size; i++) { + Tree **child = &result->children.contents[i]; if (child_start_byte > edit_byte_offset) break; - ts_tree_invalidate_lookahead(child, edit_byte_offset - child_start_byte); - child_start_byte += ts_tree_total_bytes(child); + *child = ts_tree_invalidate_lookahead(*child, edit_byte_offset - child_start_byte, pool); + child_start_byte += ts_tree_total_bytes(*child); } } + + return result; } -void ts_tree__edit(Tree *self, Edit edit) { +Tree *ts_tree__edit(Tree *self, Edit edit, TreePool *pool) { Length new_end = length_add(edit.start, edit.added); Length old_end = length_add(edit.start, edit.removed); - self->has_changes = true; + Tree *result = ts_tree_make_mut(pool, self); + result->has_changes = true; - if (old_end.bytes <= self->padding.bytes) { - self->padding = length_add(new_end, length_sub(self->padding, old_end)); - } else if (edit.start.bytes < self->padding.bytes) { - self->size = length_sub(self->size, length_sub(old_end, self->padding)); - self->padding = new_end; - } else if (edit.start.bytes == self->padding.bytes && edit.removed.bytes == 0) { - self->padding = length_add(self->padding, edit.added); + if (old_end.bytes <= result->padding.bytes) { + result->padding = length_add(new_end, length_sub(result->padding, old_end)); + } else if (edit.start.bytes < result->padding.bytes) { + result->size = length_sub(result->size, length_sub(old_end, result->padding)); + result->padding = new_end; + } else if (edit.start.bytes == result->padding.bytes && edit.removed.bytes == 0) { + result->padding = length_add(result->padding, edit.added); } else { - Length new_total_size = length_add(new_end, length_sub(ts_tree_total_size(self), old_end)); - self->size = length_sub(new_total_size, self->padding); + Length new_total_size = length_add(new_end, length_sub(ts_tree_total_size(result), old_end)); + result->size = length_sub(new_total_size, result->padding); } Length child_left, child_right = length_zero(); - for (uint32_t i = 0; i < self->children.size; i++) { - Tree *child = self->children.contents[i]; - Length child_size = ts_tree_total_size(child); + for (uint32_t i = 0; i < result->children.size; i++) { + Tree **child = &result->children.contents[i]; + Length child_size = ts_tree_total_size(*child); child_left = child_right; child_right = length_add(child_left, child_size); @@ -499,19 +519,21 @@ void ts_tree__edit(Tree *self, Edit edit) { edit.added = length_zero(); edit.removed = length_sub(edit.removed, child_edit.removed); - ts_tree__edit(child, child_edit); + *child = ts_tree__edit(*child, child_edit, pool); } else if (child_left.bytes <= edit.start.bytes) { - ts_tree_invalidate_lookahead(child, edit.start.bytes - child_left.bytes); + *child = ts_tree_invalidate_lookahead(*child, edit.start.bytes - child_left.bytes, pool); } } + + return result; } -void ts_tree_edit(Tree *self, const TSInputEdit *edit) { - ts_tree__edit(self, (Edit) { +Tree *ts_tree_edit(Tree *self, const TSInputEdit *edit, TreePool *pool) { + return ts_tree__edit(self, (Edit) { .start = {edit->start_byte, edit->start_point}, .added = {edit->bytes_added, edit->extent_added}, .removed = {edit->bytes_removed, edit->extent_removed}, - }); + }, pool); } Tree *ts_tree_last_external_token(Tree *tree) { diff --git a/src/runtime/tree.h b/src/runtime/tree.h index ce96e41d..4fa95522 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -100,7 +100,7 @@ bool ts_tree_eq(const Tree *tree1, const Tree *tree2); int ts_tree_compare(const Tree *tree1, const Tree *tree2); void ts_tree_set_children(Tree *, TreeArray *, const TSLanguage *); void ts_tree_balance(Tree *, TreePool *, const TSLanguage *); -void ts_tree_edit(Tree *, const TSInputEdit *edit); +Tree *ts_tree_edit(Tree *, const TSInputEdit *edit, TreePool *); char *ts_tree_string(const Tree *, const TSLanguage *, bool include_all); void ts_tree_print_dot_graph(const Tree *, const TSLanguage *, FILE *); Tree *ts_tree_last_external_token(Tree *); diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc index e80743e7..2d26f021 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/tree_test.cc @@ -178,7 +178,7 @@ describe("Tree", []() { }); describe("edit", [&]() { - Tree *tree = nullptr; + Tree *tree; before_each([&]() { tree = ts_tree_make_node(&pool, symbol1, tree_array({ @@ -204,7 +204,7 @@ describe("Tree", []() { edit.start_point = {0, 1}; edit.extent_removed = {0, 0}; edit.extent_added = {0, 1}; - ts_tree_edit(tree, &edit); + tree = ts_tree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -230,7 +230,7 @@ describe("Tree", []() { edit.start_point = {0, 1}; edit.extent_removed = {0, 3}; edit.extent_added = {0, 4}; - ts_tree_edit(tree, &edit); + tree = ts_tree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -252,7 +252,7 @@ describe("Tree", []() { edit.start_point = {0, 2}; edit.extent_removed = {0, 0}; edit.extent_added = {0, 2}; - ts_tree_edit(tree, &edit); + tree = ts_tree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -276,7 +276,7 @@ describe("Tree", []() { edit.start_point = {0, 2}; edit.extent_removed = {0, 2}; edit.extent_added = {0, 5}; - ts_tree_edit(tree, &edit); + tree = ts_tree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -300,7 +300,7 @@ describe("Tree", []() { edit.start_point = {0, 1}; edit.extent_removed = {0, 10}; edit.extent_added = {0, 3}; - ts_tree_edit(tree, &edit); + tree = ts_tree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -332,7 +332,7 @@ describe("Tree", []() { edit.start_point = {0, 6}; edit.extent_removed = {0, 1}; edit.extent_added = {0, 1}; - ts_tree_edit(tree, &edit); + tree = ts_tree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->children.contents[0]->has_changes, IsTrue()); From 61327b627a575e2d47968fc2ac68a2073c4407f1 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 10 May 2018 12:28:16 -0700 Subject: [PATCH 17/31] Add a unit test asserting that ts_tree_edit doesn't mutate the tree Co-Authored-By: Rick Winfrey --- test/runtime/tree_test.cc | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc index 2d26f021..1613b930 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/tree_test.cc @@ -195,6 +195,35 @@ describe("Tree", []() { ts_tree_release(&pool, tree); }); + it("does not mutate the argument", [&]() { + TSInputEdit edit; + edit.start_byte = 1; + edit.bytes_removed = 0; + edit.bytes_added = 1; + edit.start_point = {0, 1}; + edit.extent_removed = {0, 0}; + edit.extent_added = {0, 1}; + + ts_tree_retain(tree); + Tree *new_tree = ts_tree_edit(tree, &edit, &pool); + assert_consistent(tree); + assert_consistent(new_tree); + + AssertThat(tree->has_changes, IsFalse()); + AssertThat(tree->padding, Equals({2, {0, 2}})); + AssertThat(tree->size, Equals({13, {0, 13}})); + + AssertThat(tree->children.contents[0]->has_changes, IsFalse()); + AssertThat(tree->children.contents[0]->padding, Equals({2, {0, 2}})); + AssertThat(tree->children.contents[0]->size, Equals({3, {0, 3}})); + + AssertThat(tree->children.contents[1]->has_changes, IsFalse()); + AssertThat(tree->children.contents[1]->padding, Equals({2, {0, 2}})); + AssertThat(tree->children.contents[1]->size, Equals({3, {0, 3}})); + + ts_tree_release(&pool, new_tree); + }); + describe("edits within a tree's padding", [&]() { it("resizes the padding of the tree and its leftmost descendants", [&]() { TSInputEdit edit; From 35510a612de0a458c2cca3640294eecb6fcb552c Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 10 May 2018 15:11:14 -0700 Subject: [PATCH 18/31] Rename Tree -> Subtree --- project.gyp | 2 +- src/runtime/document.c | 22 +- src/runtime/document.h | 4 +- src/runtime/get_changed_ranges.c | 22 +- src/runtime/get_changed_ranges.h | 6 +- src/runtime/language.c | 2 +- src/runtime/language.h | 2 +- src/runtime/lexer.c | 2 +- src/runtime/lexer.h | 2 +- src/runtime/node.c | 32 +-- src/runtime/parser.c | 204 ++++++++--------- src/runtime/parser.h | 14 +- src/runtime/reusable_node.h | 16 +- src/runtime/stack.c | 172 +++++++------- src/runtime/stack.h | 14 +- src/runtime/{tree.c => subtree.c} | 215 ++++++++--------- src/runtime/subtree.h | 125 ++++++++++ src/runtime/tree.h | 125 ---------- src/runtime/tree_cursor.c | 12 +- src/runtime/tree_cursor.h | 4 +- test/helpers/tree_helpers.cc | 12 +- test/helpers/tree_helpers.h | 8 +- test/runtime/stack_test.cc | 216 +++++++++--------- .../runtime/{tree_test.cc => subtree_test.cc} | 208 ++++++++--------- tests.gyp | 2 +- 25 files changed, 724 insertions(+), 719 deletions(-) rename src/runtime/{tree.c => subtree.c} (70%) create mode 100644 src/runtime/subtree.h delete mode 100644 src/runtime/tree.h rename test/runtime/{tree_test.cc => subtree_test.cc} (65%) diff --git a/project.gyp b/project.gyp index f59c3c91..cfc1776b 100644 --- a/project.gyp +++ b/project.gyp @@ -95,7 +95,7 @@ 'src/runtime/stack.c', 'src/runtime/parser.c', 'src/runtime/string_input.c', - 'src/runtime/tree.c', + 'src/runtime/subtree.c', 'src/runtime/tree_cursor.c', 'src/runtime/utf16.c', 'externals/utf8proc/utf8proc.c', diff --git a/src/runtime/document.c b/src/runtime/document.c index 9a13ca30..4298025a 100644 --- a/src/runtime/document.c +++ b/src/runtime/document.c @@ -1,5 +1,5 @@ #include "runtime/alloc.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/parser.h" #include "runtime/string_input.h" #include "runtime/document.h" @@ -19,7 +19,7 @@ TSDocument *ts_document_new() { } void ts_document_free(TSDocument *self) { - if (self->tree) ts_tree_release(&self->parser.tree_pool, self->tree); + if (self->tree) ts_subtree_release(&self->parser.tree_pool, self->tree); if (self->cursor1.stack.contents) array_delete(&self->cursor1.stack); if (self->cursor2.stack.contents) array_delete(&self->cursor2.stack); parser_destroy(&self->parser); @@ -41,7 +41,7 @@ void ts_document_set_language(TSDocument *self, const TSLanguage *language) { ts_document_invalidate(self); parser_set_language(&self->parser, language); if (self->tree) { - ts_tree_release(&self->parser.tree_pool, self->tree); + ts_subtree_release(&self->parser.tree_pool, self->tree); self->tree = NULL; } } @@ -91,16 +91,16 @@ void ts_document_edit(TSDocument *self, TSInputEdit edit) { if (!self->tree) return; - uint32_t max_bytes = ts_tree_total_bytes(self->tree); + uint32_t max_bytes = ts_subtree_total_bytes(self->tree); if (edit.start_byte > max_bytes) return; if (edit.bytes_removed > max_bytes - edit.start_byte) edit.bytes_removed = max_bytes - edit.start_byte; - self->tree = ts_tree_edit(self->tree, &edit, &self->parser.tree_pool); + self->tree = ts_subtree_edit(self->tree, &edit, &self->parser.tree_pool); if (self->parser.print_debugging_graphs) { - ts_tree_print_dot_graph(self->tree, self->parser.language, stderr); + ts_subtree_print_dot_graph(self->tree, self->parser.language, stderr); } } @@ -130,18 +130,18 @@ void ts_document_parse_with_options(TSDocument *self, TSParseOptions options) { if (!self->input.read || !self->parser.language) return; - Tree *reusable_tree = self->valid ? self->tree : NULL; + Subtree *reusable_tree = self->valid ? self->tree : NULL; if (reusable_tree && !reusable_tree->has_changes) return; - Tree *tree = parser_parse(&self->parser, self->input, reusable_tree, options.halt_on_error); + Subtree *tree = parser_parse(&self->parser, self->input, reusable_tree, options.halt_on_error); if (self->tree) { - Tree *old_tree = self->tree; + Subtree *old_tree = self->tree; self->tree = tree; if (options.changed_ranges && options.changed_range_count) { - *options.changed_range_count = ts_tree_get_changed_ranges( + *options.changed_range_count = ts_subtree_get_changed_ranges( old_tree, tree, &self->cursor1, &self->cursor2, self->parser.language, options.changed_ranges ); @@ -158,7 +158,7 @@ void ts_document_parse_with_options(TSDocument *self, TSParseOptions options) { } } - ts_tree_release(&self->parser.tree_pool, old_tree); + ts_subtree_release(&self->parser.tree_pool, old_tree); } self->tree = tree; diff --git a/src/runtime/document.h b/src/runtime/document.h index 986fcec9..9e53963b 100644 --- a/src/runtime/document.h +++ b/src/runtime/document.h @@ -2,14 +2,14 @@ #define RUNTIME_DOCUMENT_H_ #include "runtime/parser.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/tree_cursor.h" #include struct TSDocument { Parser parser; TSInput input; - Tree *tree; + Subtree *tree; TSTreeCursor cursor1; TSTreeCursor cursor2; size_t parse_count; diff --git a/src/runtime/get_changed_ranges.c b/src/runtime/get_changed_ranges.c index 7632af24..76e01f29 100644 --- a/src/runtime/get_changed_ranges.c +++ b/src/runtime/get_changed_ranges.c @@ -1,5 +1,5 @@ #include "runtime/get_changed_ranges.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/language.h" #include "runtime/error_costs.h" #include "runtime/tree_cursor.h" @@ -31,7 +31,7 @@ typedef struct { bool in_padding; } Iterator; -static Iterator iterator_new(TSTreeCursor *cursor, Tree *tree, const TSLanguage *language) { +static Iterator iterator_new(TSTreeCursor *cursor, Subtree *tree, const TSLanguage *language) { array_clear(&cursor->stack); array_push(&cursor->stack, ((TreeCursorEntry){ .tree = tree, @@ -74,14 +74,14 @@ static bool iterator_tree_is_visible(const Iterator *self) { TreeCursorEntry entry = *array_back(&self->cursor.stack); if (entry.tree->visible) return true; if (self->cursor.stack.size > 1) { - Tree *parent = self->cursor.stack.contents[self->cursor.stack.size - 2].tree; + Subtree *parent = self->cursor.stack.contents[self->cursor.stack.size - 2].tree; const TSSymbol *alias_sequence = ts_language_alias_sequence(self->language, parent->alias_sequence_id); return alias_sequence && alias_sequence[entry.structural_child_index] != 0; } return false; } -static void iterator_get_visible_state(const Iterator *self, Tree **tree, +static void iterator_get_visible_state(const Iterator *self, Subtree **tree, TSSymbol *alias_symbol, uint32_t *start_byte) { uint32_t i = self->cursor.stack.size - 1; @@ -94,7 +94,7 @@ static void iterator_get_visible_state(const Iterator *self, Tree **tree, TreeCursorEntry entry = self->cursor.stack.contents[i]; if (i > 0) { - Tree *parent = self->cursor.stack.contents[i - 1].tree; + Subtree *parent = self->cursor.stack.contents[i - 1].tree; const TSSymbol *alias_sequence = ts_language_alias_sequence( self->language, parent->alias_sequence_id @@ -129,7 +129,7 @@ static bool iterator_descend(Iterator *self, uint32_t goal_position) { Length position = entry.position; uint32_t structural_child_index = 0; for (uint32_t i = 0; i < entry.tree->children.size; i++) { - Tree *child = entry.tree->children.contents[i]; + Subtree *child = entry.tree->children.contents[i]; Length child_left = length_add(position, child->padding); Length child_right = length_add(child_left, child->size); @@ -178,13 +178,13 @@ static void iterator_advance(Iterator *self) { TreeCursorEntry entry = array_pop(&self->cursor.stack); if (iterator_done(self)) return; - Tree *parent = array_back(&self->cursor.stack)->tree; + Subtree *parent = array_back(&self->cursor.stack)->tree; uint32_t child_index = entry.child_index + 1; if (parent->children.size > child_index) { - Length position = length_add(entry.position, ts_tree_total_size(entry.tree)); + Length position = length_add(entry.position, ts_subtree_total_size(entry.tree)); uint32_t structural_child_index = entry.structural_child_index; if (!entry.tree->extra) structural_child_index++; - Tree *next_child = parent->children.contents[child_index]; + Subtree *next_child = parent->children.contents[child_index]; array_push(&self->cursor.stack, ((TreeCursorEntry){ .tree = next_child, @@ -214,7 +214,7 @@ typedef enum { } IteratorComparison; IteratorComparison iterator_compare(const Iterator *old_iter, const Iterator *new_iter) { - Tree *old_tree = NULL, *new_tree = NULL; + Subtree *old_tree = NULL, *new_tree = NULL; uint32_t old_start = 0, new_start = 0; TSSymbol old_alias_symbol = 0, new_alias_symbol = 0; iterator_get_visible_state(old_iter, &old_tree, &old_alias_symbol, &old_start); @@ -261,7 +261,7 @@ static inline void iterator_print_state(Iterator *self) { } #endif -unsigned ts_tree_get_changed_ranges(Tree *old_tree, Tree *new_tree, +unsigned ts_subtree_get_changed_ranges(Subtree *old_tree, Subtree *new_tree, TSTreeCursor *cursor1, TSTreeCursor *cursor2, const TSLanguage *language, TSRange **ranges) { RangeArray results = array_new(); diff --git a/src/runtime/get_changed_ranges.h b/src/runtime/get_changed_ranges.h index fa408aaa..bf138a21 100644 --- a/src/runtime/get_changed_ranges.h +++ b/src/runtime/get_changed_ranges.h @@ -1,10 +1,10 @@ #ifndef RUNTIME_GET_CHANGED_RANGES_H_ #define RUNTIME_GET_CHANGED_RANGES_H_ -#include "runtime/tree.h" +#include "runtime/subtree.h" -unsigned ts_tree_get_changed_ranges( - Tree *old_tree, Tree *new_tree, TSTreeCursor *cursor1, TSTreeCursor *cursor2, +unsigned ts_subtree_get_changed_ranges( + Subtree *old_tree, Subtree *new_tree, TSTreeCursor *cursor1, TSTreeCursor *cursor2, const TSLanguage *language, TSRange **ranges ); diff --git a/src/runtime/language.c b/src/runtime/language.c index 9bf1fc63..75e7da8b 100644 --- a/src/runtime/language.c +++ b/src/runtime/language.c @@ -1,5 +1,5 @@ #include "runtime/language.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/error_costs.h" void ts_language_table_entry(const TSLanguage *self, TSStateId state, diff --git a/src/runtime/language.h b/src/runtime/language.h index 64733242..879a5b5b 100644 --- a/src/runtime/language.h +++ b/src/runtime/language.h @@ -6,7 +6,7 @@ extern "C" { #endif #include "tree_sitter/parser.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1) diff --git a/src/runtime/lexer.c b/src/runtime/lexer.c index 157276a1..e6e5aa58 100644 --- a/src/runtime/lexer.c +++ b/src/runtime/lexer.c @@ -1,6 +1,6 @@ #include #include "runtime/lexer.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/length.h" #include "runtime/utf16.h" #include "utf8proc.h" diff --git a/src/runtime/lexer.h b/src/runtime/lexer.h index 08a71a11..90be55a7 100644 --- a/src/runtime/lexer.h +++ b/src/runtime/lexer.h @@ -8,7 +8,7 @@ extern "C" { #include "tree_sitter/parser.h" #include "tree_sitter/runtime.h" #include "runtime/length.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" typedef struct { TSLexer data; diff --git a/src/runtime/node.c b/src/runtime/node.c index 77d913f3..4df3400a 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -1,12 +1,12 @@ #include -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/document.h" #include "runtime/language.h" // NodeChildIterator typedef struct { - const Tree *parent; + const Subtree *parent; const TSDocument *document; Length position; uint32_t child_index; @@ -25,12 +25,12 @@ static inline TSNode ts_node__null() { }; } -static inline const Tree *ts_node__tree(TSNode self) { +static inline const Subtree *ts_node__tree(TSNode self) { return self.subtree; } static inline NodeChildIterator ts_node_child_iterator_begin(const TSNode *node) { - const Tree *tree = ts_node__tree(*node); + const Subtree *tree = ts_node__tree(*node); const TSSymbol *alias_sequence = ts_language_alias_sequence( node->document->parser.language, tree->alias_sequence_id @@ -47,7 +47,7 @@ static inline NodeChildIterator ts_node_child_iterator_begin(const TSNode *node) static inline bool ts_node_child_iterator_next(NodeChildIterator *self, TSNode *result) { if (self->child_index == self->parent->children.size) return false; - Tree *child = self->parent->children.contents[self->child_index]; + Subtree *child = self->parent->children.contents[self->child_index]; TSSymbol alias_symbol = 0; if (!child->extra) { if (self->alias_sequence) { @@ -62,13 +62,13 @@ static inline bool ts_node_child_iterator_next(NodeChildIterator *self, TSNode * .byte = self->position.bytes, .alias_symbol = alias_symbol, }; - self->position = length_add(self->position, ts_tree_total_size(child)); + self->position = length_add(self->position, ts_subtree_total_size(child)); self->child_index++; return true; } static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { - const Tree *tree = ts_node__tree(self); + const Subtree *tree = ts_node__tree(self); if (include_anonymous) { return tree->visible || self.alias_symbol; } else { @@ -86,7 +86,7 @@ static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { } static inline uint32_t ts_node__relevant_child_count(TSNode self, bool include_anonymous) { - const Tree *tree = ts_node__tree(self); + const Subtree *tree = ts_node__tree(self); if (tree->children.size > 0) { if (include_anonymous) { return tree->visible_child_count; @@ -300,7 +300,7 @@ static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint mi TSNode child; NodeChildIterator iterator = ts_node_child_iterator_begin(&node); while (ts_node_child_iterator_next(&iterator, &child)) { - const Tree *child_tree = ts_node__tree(child); + const Subtree *child_tree = ts_node__tree(child); if (iterator.child_index != 1) { start_position = point_add(start_position, child_tree->padding.extent); } @@ -338,7 +338,7 @@ TSPoint ts_node_end_point(TSNode self) { } TSSymbol ts_node_symbol(TSNode self) { - const Tree *tree = ts_node__tree(self); + const Subtree *tree = ts_node__tree(self); return self.alias_symbol ? self.alias_symbol : tree->symbol; } @@ -347,25 +347,25 @@ const char *ts_node_type(TSNode self) { } char *ts_node_string(TSNode self) { - return ts_tree_string(ts_node__tree(self), self.document->parser.language, false); + return ts_subtree_string(ts_node__tree(self), self.document->parser.language, false); } bool ts_node_eq(TSNode self, TSNode other) { return ( - ts_tree_eq(ts_node__tree(self), ts_node__tree(other)) && + ts_subtree_eq(ts_node__tree(self), ts_node__tree(other)) && self.byte == other.byte ); } bool ts_node_is_named(TSNode self) { - const Tree *tree = ts_node__tree(self); + const Subtree *tree = ts_node__tree(self); return self.alias_symbol ? ts_language_symbol_metadata(self.document->parser.language, self.alias_symbol).named : tree->named; } bool ts_node_is_missing(TSNode self) { - const Tree *tree = ts_node__tree(self); + const Subtree *tree = ts_node__tree(self); return tree->is_missing; } @@ -414,7 +414,7 @@ TSNode ts_node_named_child(TSNode self, uint32_t child_index) { } uint32_t ts_node_child_count(TSNode self) { - const Tree *tree = ts_node__tree(self); + const Subtree *tree = ts_node__tree(self); if (tree->children.size > 0) { return tree->visible_child_count; } else { @@ -423,7 +423,7 @@ uint32_t ts_node_child_count(TSNode self) { } uint32_t ts_node_named_child_count(TSNode self) { - const Tree *tree = ts_node__tree(self); + const Subtree *tree = ts_node__tree(self); if (tree->children.size > 0) { return tree->named_child_count; } else { diff --git a/src/runtime/parser.c b/src/runtime/parser.c index f1ba0d16..8bb7ae31 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -4,7 +4,7 @@ #include #include #include "tree_sitter/runtime.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/lexer.h" #include "runtime/length.h" #include "runtime/array.h" @@ -27,7 +27,7 @@ #define LOG_TREE() \ if (self->print_debugging_graphs) { \ - ts_tree_print_dot_graph(self->finished_tree, self->language, stderr); \ + ts_subtree_print_dot_graph(self->finished_tree, self->language, stderr); \ fputs("\n", stderr); \ } @@ -84,10 +84,10 @@ static bool parser__breakdown_top_of_stack(Parser *self, StackVersion version) { for (uint32_t i = 0; i < pop.size; i++) { StackSlice slice = pop.contents[i]; TSStateId state = ts_stack_state(self->stack, slice.version); - Tree *parent = *array_front(&slice.trees); + Subtree *parent = *array_front(&slice.subtrees); for (uint32_t j = 0; j < parent->children.size; j++) { - Tree *child = parent->children.contents[j]; + Subtree *child = parent->children.contents[j]; pending = child->children.size > 0; if (child->symbol == ts_builtin_sym_error) { @@ -96,17 +96,17 @@ static bool parser__breakdown_top_of_stack(Parser *self, StackVersion version) { state = ts_language_next_state(self->language, state, child->symbol); } - ts_tree_retain(child); + ts_subtree_retain(child); ts_stack_push(self->stack, slice.version, child, pending, state); } - for (uint32_t j = 1; j < slice.trees.size; j++) { - Tree *tree = slice.trees.contents[j]; + for (uint32_t j = 1; j < slice.subtrees.size; j++) { + Subtree *tree = slice.subtrees.contents[j]; ts_stack_push(self->stack, slice.version, tree, false, state); } - ts_tree_release(&self->tree_pool, parent); - array_delete(&slice.trees); + ts_subtree_release(&self->tree_pool, parent); + array_delete(&slice.subtrees); LOG("breakdown_top_of_stack tree:%s", SYM_NAME(parent->symbol)); LOG_STACK(); @@ -116,11 +116,11 @@ static bool parser__breakdown_top_of_stack(Parser *self, StackVersion version) { return did_break_down; } -static void parser__breakdown_lookahead(Parser *self, Tree **lookahead, +static void parser__breakdown_lookahead(Parser *self, Subtree **lookahead, TSStateId state, ReusableNode *reusable_node) { bool did_descend = false; - Tree *tree = reusable_node_tree(reusable_node); + Subtree *tree = reusable_node_tree(reusable_node); while (tree->children.size > 0 && tree->parse_state != state) { LOG("state_mismatch sym:%s", SYM_NAME(tree->symbol)); reusable_node_descend(reusable_node); @@ -129,9 +129,9 @@ static void parser__breakdown_lookahead(Parser *self, Tree **lookahead, } if (did_descend) { - ts_tree_release(&self->tree_pool, *lookahead); + ts_subtree_release(&self->tree_pool, *lookahead); *lookahead = tree; - ts_tree_retain(*lookahead); + ts_subtree_retain(*lookahead); } } @@ -215,7 +215,7 @@ static bool parser__better_version_exists(Parser *self, StackVersion version, return false; } -static void parser__restore_external_scanner(Parser *self, Tree *external_token) { +static void parser__restore_external_scanner(Parser *self, Subtree *external_token) { if (external_token) { self->language->external_scanner.deserialize( self->external_scanner_payload, @@ -227,9 +227,9 @@ static void parser__restore_external_scanner(Parser *self, Tree *external_token) } } -static Tree *parser__lex(Parser *self, StackVersion version, TSStateId parse_state) { +static Subtree *parser__lex(Parser *self, StackVersion version, TSStateId parse_state) { Length start_position = ts_stack_position(self->stack, version); - Tree *external_token = ts_stack_last_external_token(self->stack, version); + Subtree *external_token = ts_stack_last_external_token(self->stack, version); TSLexMode lex_mode = self->language->lex_modes[parse_state]; const bool *valid_external_tokens = ts_language_enabled_external_tokens( self->language, @@ -326,11 +326,11 @@ static Tree *parser__lex(Parser *self, StackVersion version, TSStateId parse_sta last_byte_scanned = self->lexer.current_position.bytes; } - Tree *result; + Subtree *result; if (skipped_error) { Length padding = length_sub(error_start_position, start_position); Length size = length_sub(error_end_position, error_start_position); - result = ts_tree_make_error(&self->tree_pool, size, padding, first_error_character, self->language); + result = ts_subtree_make_error(&self->tree_pool, size, padding, first_error_character, self->language); } else { if (self->lexer.token_end_position.bytes < self->lexer.token_start_position.bytes) { self->lexer.token_start_position = self->lexer.token_end_position; @@ -355,7 +355,7 @@ static Tree *parser__lex(Parser *self, StackVersion version, TSStateId parse_sta } } - result = ts_tree_make_leaf(&self->tree_pool, symbol, padding, size, self->language); + result = ts_subtree_make_leaf(&self->tree_pool, symbol, padding, size, self->language); if (found_external_token) { result->has_external_tokens = true; @@ -375,30 +375,30 @@ static Tree *parser__lex(Parser *self, StackVersion version, TSStateId parse_sta return result; } -static Tree *parser__get_cached_token(Parser *self, size_t byte_index, Tree *last_external_token) { +static Subtree *parser__get_cached_token(Parser *self, size_t byte_index, Subtree *last_external_token) { TokenCache *cache = &self->token_cache; if (cache->token && cache->byte_index == byte_index && - ts_tree_external_token_state_eq(cache->last_external_token, last_external_token)) { + ts_subtree_external_token_state_eq(cache->last_external_token, last_external_token)) { return cache->token; } else { return NULL; } } -static void parser__set_cached_token(Parser *self, size_t byte_index, Tree *last_external_token, - Tree *token) { +static void parser__set_cached_token(Parser *self, size_t byte_index, Subtree *last_external_token, + Subtree *token) { TokenCache *cache = &self->token_cache; - if (token) ts_tree_retain(token); - if (last_external_token) ts_tree_retain(last_external_token); - if (cache->token) ts_tree_release(&self->tree_pool, cache->token); - if (cache->last_external_token) ts_tree_release(&self->tree_pool, cache->last_external_token); + if (token) ts_subtree_retain(token); + if (last_external_token) ts_subtree_retain(last_external_token); + if (cache->token) ts_subtree_release(&self->tree_pool, cache->token); + if (cache->last_external_token) ts_subtree_release(&self->tree_pool, cache->last_external_token); cache->token = token; cache->byte_index = byte_index; cache->last_external_token = last_external_token; } -static bool parser__can_reuse_first_leaf(Parser *self, TSStateId state, Tree *tree, +static bool parser__can_reuse_first_leaf(Parser *self, TSStateId state, Subtree *tree, TableEntry *table_entry) { TSLexMode current_lex_mode = self->language->lex_modes[state]; @@ -416,12 +416,12 @@ static bool parser__can_reuse_first_leaf(Parser *self, TSStateId state, Tree *tr return current_lex_mode.external_lex_state == 0 && table_entry->is_reusable; } -static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId *state, +static Subtree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId *state, ReusableNode *reusable_node, TableEntry *table_entry) { Length position = ts_stack_position(self->stack, version); - Tree *last_external_token = ts_stack_last_external_token(self->stack, version); + Subtree *last_external_token = ts_stack_last_external_token(self->stack, version); - Tree *result; + Subtree *result; while ((result = reusable_node_tree(reusable_node))) { uint32_t byte_offset = reusable_node_byte_offset(reusable_node); if (byte_offset > position.bytes) { @@ -435,7 +435,7 @@ static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId continue; } - if (!ts_tree_external_token_state_eq(reusable_node->last_external_token, last_external_token)) { + if (!ts_subtree_external_token_state_eq(reusable_node->last_external_token, last_external_token)) { LOG("reusable_node_has_different_external_scanner_state symbol:%s", SYM_NAME(result->symbol)); reusable_node_advance(reusable_node); continue; @@ -476,14 +476,14 @@ static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId } LOG("reuse_node symbol:%s", SYM_NAME(result->symbol)); - ts_tree_retain(result); + ts_subtree_retain(result); return result; } if ((result = parser__get_cached_token(self, position.bytes, last_external_token))) { ts_language_table_entry(self->language, *state, result->first_leaf.symbol, table_entry); if (parser__can_reuse_first_leaf(self, *state, result, table_entry)) { - ts_tree_retain(result); + ts_subtree_retain(result); return result; } } @@ -494,7 +494,7 @@ static Tree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId return result; } -static bool parser__select_tree(Parser *self, Tree *left, Tree *right) { +static bool parser__select_tree(Parser *self, Subtree *left, Subtree *right) { if (!left) return true; if (!right) return false; @@ -526,7 +526,7 @@ static bool parser__select_tree(Parser *self, Tree *left, Tree *right) { if (left->error_cost > 0) return true; - int comparison = ts_tree_compare(left, right); + int comparison = ts_subtree_compare(left, right); switch (comparison) { case -1: LOG("select_earlier symbol:%s, over_symbol:%s", SYM_NAME(left->symbol), @@ -545,31 +545,31 @@ static bool parser__select_tree(Parser *self, Tree *left, Tree *right) { } static void parser__shift(Parser *self, StackVersion version, TSStateId state, - Tree *lookahead, bool extra) { + Subtree *lookahead, bool extra) { if (extra != lookahead->extra) { if (ts_stack_version_count(self->stack) > 1) { - lookahead = ts_tree_make_copy(&self->tree_pool, lookahead); + lookahead = ts_subtree_make_copy(&self->tree_pool, lookahead); } else { - ts_tree_retain(lookahead); + ts_subtree_retain(lookahead); } lookahead->extra = extra; } else { - ts_tree_retain(lookahead); + ts_subtree_retain(lookahead); } bool is_pending = lookahead->children.size > 0; ts_stack_push(self->stack, version, lookahead, is_pending, state); if (lookahead->has_external_tokens) { ts_stack_set_last_external_token( - self->stack, version, ts_tree_last_external_token(lookahead) + self->stack, version, ts_subtree_last_external_token(lookahead) ); } } -static bool parser__replace_children(Parser *self, Tree *tree, TreeArray *children) { +static bool parser__replace_children(Parser *self, Subtree *tree, SubtreeArray *children) { self->scratch_tree = *tree; self->scratch_tree.children.size = 0; - ts_tree_set_children(&self->scratch_tree, children, self->language); + ts_subtree_set_children(&self->scratch_tree, children, self->language); if (parser__select_tree(self, tree, &self->scratch_tree)) { *tree = self->scratch_tree; return true; @@ -591,12 +591,12 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb // Extra tokens on top of the stack should not be included in this new parent // node. They will be re-pushed onto the stack after the parent node is // created and pushed. - TreeArray children = slice.trees; + SubtreeArray children = slice.subtrees; while (children.size > 0 && children.contents[children.size - 1]->extra) { children.size--; } - Tree *parent = ts_tree_make_node(&self->tree_pool, + Subtree *parent = ts_subtree_make_node(&self->tree_pool, symbol, &children, alias_sequence_id, self->language ); @@ -609,16 +609,16 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb if (next_slice.version != slice.version) break; i++; - TreeArray children = next_slice.trees; + SubtreeArray children = next_slice.subtrees; while (children.size > 0 && children.contents[children.size - 1]->extra) { children.size--; } if (parser__replace_children(self, parent, &children)) { - ts_tree_array_delete(&self->tree_pool, &slice.trees); + ts_subtree_array_delete(&self->tree_pool, &slice.subtrees); slice = next_slice; } else { - ts_tree_array_delete(&self->tree_pool, &next_slice.trees); + ts_subtree_array_delete(&self->tree_pool, &next_slice.subtrees); } } @@ -638,15 +638,15 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb // Push the parent node onto the stack, along with any extra tokens that // were previously on top of the stack. ts_stack_push(self->stack, slice.version, parent, false, next_state); - for (uint32_t j = parent->children.size; j < slice.trees.size; j++) { - ts_stack_push(self->stack, slice.version, slice.trees.contents[j], false, next_state); + for (uint32_t j = parent->children.size; j < slice.subtrees.size; j++) { + ts_stack_push(self->stack, slice.version, slice.subtrees.contents[j], false, next_state); } if (ts_stack_version_count(self->stack) > MAX_VERSION_COUNT) { i++; while (i < pop.size) { StackSlice slice = pop.contents[i]; - ts_tree_array_delete(&self->tree_pool, &slice.trees); + ts_subtree_array_delete(&self->tree_pool, &slice.subtrees); ts_stack_halt(self->stack, slice.version); i++; } @@ -669,7 +669,7 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb return pop; } -static void parser__start(Parser *self, TSInput input, Tree *previous_tree) { +static void parser__start(Parser *self, TSInput input, Subtree *previous_tree) { if (previous_tree) { LOG("parse_after_edit"); } else { @@ -688,29 +688,29 @@ static void parser__start(Parser *self, TSInput input, Tree *previous_tree) { self->in_ambiguity = false; } -static void parser__accept(Parser *self, StackVersion version, Tree *lookahead) { +static void parser__accept(Parser *self, StackVersion version, Subtree *lookahead) { lookahead->extra = true; assert(lookahead->symbol == ts_builtin_sym_end); - ts_tree_retain(lookahead); + ts_subtree_retain(lookahead); ts_stack_push(self->stack, version, lookahead, false, 1); StackSliceArray pop = ts_stack_pop_all(self->stack, version); for (uint32_t i = 0; i < pop.size; i++) { - TreeArray trees = pop.contents[i].trees; + SubtreeArray trees = pop.contents[i].subtrees; - Tree *root = NULL; + Subtree *root = NULL; for (uint32_t j = trees.size - 1; j + 1 > 0; j--) { - Tree *child = trees.contents[j]; + Subtree *child = trees.contents[j]; if (!child->extra) { for (uint32_t k = 0; k < child->children.size; k++) { - ts_tree_retain(child->children.contents[k]); + ts_subtree_retain(child->children.contents[k]); } array_splice(&trees, j, 1, &child->children); - root = ts_tree_make_node( + root = ts_subtree_make_node( &self->tree_pool, child->symbol, &trees, child->alias_sequence_id, self->language ); - ts_tree_release(&self->tree_pool, child); + ts_subtree_release(&self->tree_pool, child); break; } } @@ -720,10 +720,10 @@ static void parser__accept(Parser *self, StackVersion version, Tree *lookahead) if (self->finished_tree) { if (parser__select_tree(self, self->finished_tree, root)) { - ts_tree_release(&self->tree_pool, self->finished_tree); + ts_subtree_release(&self->tree_pool, self->finished_tree); self->finished_tree = root; } else { - ts_tree_release(&self->tree_pool, root); + ts_subtree_release(&self->tree_pool, root); } } else { self->finished_tree = root; @@ -845,7 +845,7 @@ static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lo lookahead_symbol )) { StackVersion version_with_missing_tree = ts_stack_copy_version(self->stack, v); - Tree *missing_tree = ts_tree_make_missing_leaf(&self->tree_pool, missing_symbol, self->language); + Subtree *missing_tree = ts_subtree_make_missing_leaf(&self->tree_pool, missing_symbol, self->language); ts_stack_push( self->stack, version_with_missing_tree, missing_tree, false, @@ -890,17 +890,17 @@ static void parser__halt_parse(Parser *self) { ts_stack_position(self->stack, 0) ); - Tree *filler_node = ts_tree_make_error(&self->tree_pool, remaining_length, length_zero(), 0, self->language); + Subtree *filler_node = ts_subtree_make_error(&self->tree_pool, remaining_length, length_zero(), 0, self->language); filler_node->visible = false; ts_stack_push(self->stack, 0, filler_node, false, 0); - TreeArray children = array_new(); - Tree *root_error = ts_tree_make_error_node(&self->tree_pool, &children, self->language); + SubtreeArray children = array_new(); + Subtree *root_error = ts_subtree_make_error_node(&self->tree_pool, &children, self->language); ts_stack_push(self->stack, 0, root_error, false, 0); - Tree *eof = ts_tree_make_leaf(&self->tree_pool, ts_builtin_sym_end, length_zero(), length_zero(), self->language); + Subtree *eof = ts_subtree_make_leaf(&self->tree_pool, ts_builtin_sym_end, length_zero(), length_zero(), self->language); parser__accept(self, 0, eof); - ts_tree_release(&self->tree_pool, eof); + ts_subtree_release(&self->tree_pool, eof); } static bool parser__recover_to_state(Parser *self, StackVersion version, unsigned depth, @@ -912,40 +912,40 @@ static bool parser__recover_to_state(Parser *self, StackVersion version, unsigne StackSlice slice = pop.contents[i]; if (slice.version == previous_version) { - ts_tree_array_delete(&self->tree_pool, &slice.trees); + ts_subtree_array_delete(&self->tree_pool, &slice.subtrees); array_erase(&pop, i--); continue; } if (ts_stack_state(self->stack, slice.version) != goal_state) { ts_stack_halt(self->stack, slice.version); - ts_tree_array_delete(&self->tree_pool, &slice.trees); + ts_subtree_array_delete(&self->tree_pool, &slice.subtrees); array_erase(&pop, i--); continue; } - TreeArray error_trees = ts_stack_pop_error(self->stack, slice.version); + SubtreeArray error_trees = ts_stack_pop_error(self->stack, slice.version); if (error_trees.size > 0) { assert(error_trees.size == 1); - array_splice(&slice.trees, 0, 0, &error_trees.contents[0]->children); + array_splice(&slice.subtrees, 0, 0, &error_trees.contents[0]->children); for (unsigned j = 0; j < error_trees.contents[0]->children.size; j++) { - ts_tree_retain(slice.trees.contents[j]); + ts_subtree_retain(slice.subtrees.contents[j]); } - ts_tree_array_delete(&self->tree_pool, &error_trees); + ts_subtree_array_delete(&self->tree_pool, &error_trees); } - TreeArray trailing_extras = ts_tree_array_remove_trailing_extras(&slice.trees); + SubtreeArray trailing_extras = ts_subtree_array_remove_trailing_extras(&slice.subtrees); - if (slice.trees.size > 0) { - Tree *error = ts_tree_make_error_node(&self->tree_pool, &slice.trees, self->language); + if (slice.subtrees.size > 0) { + Subtree *error = ts_subtree_make_error_node(&self->tree_pool, &slice.subtrees, self->language); error->extra = true; ts_stack_push(self->stack, slice.version, error, false, goal_state); } else { - array_delete(&slice.trees); + array_delete(&slice.subtrees); } for (unsigned j = 0; j < trailing_extras.size; j++) { - Tree *tree = trailing_extras.contents[j]; + Subtree *tree = trailing_extras.contents[j]; ts_stack_push(self->stack, slice.version, tree, false, goal_state); } @@ -956,7 +956,7 @@ static bool parser__recover_to_state(Parser *self, StackVersion version, unsigne return previous_version != STACK_VERSION_NONE; } -static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) { +static void parser__recover(Parser *self, StackVersion version, Subtree *lookahead) { bool did_recover = false; unsigned previous_version_count = ts_stack_version_count(self->stack); Length position = ts_stack_position(self->stack, version); @@ -1017,8 +1017,8 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) if (lookahead->symbol == ts_builtin_sym_end) { LOG("recover_eof"); - TreeArray children = array_new(); - Tree *parent = ts_tree_make_error_node(&self->tree_pool, &children, self->language); + SubtreeArray children = array_new(); + Subtree *parent = ts_subtree_make_error_node(&self->tree_pool, &children, self->language); ts_stack_push(self->stack, version, parent, false, 1); parser__accept(self, version, lookahead); return; @@ -1026,8 +1026,8 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) unsigned new_cost = current_error_cost + ERROR_COST_PER_SKIPPED_TREE + - ts_tree_total_bytes(lookahead) * ERROR_COST_PER_SKIPPED_CHAR + - ts_tree_total_size(lookahead).extent.row * ERROR_COST_PER_SKIPPED_LINE; + ts_subtree_total_bytes(lookahead) * ERROR_COST_PER_SKIPPED_CHAR + + ts_subtree_total_size(lookahead).extent.row * ERROR_COST_PER_SKIPPED_LINE; if (parser__better_version_exists(self, version, false, new_cost)) { ts_stack_halt(self->stack, version); @@ -1041,11 +1041,11 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) } LOG("skip_token symbol:%s", SYM_NAME(lookahead->symbol)); - ts_tree_retain(lookahead); - TreeArray children = array_new(); + ts_subtree_retain(lookahead); + SubtreeArray children = array_new(); array_reserve(&children, 1); array_push(&children, lookahead); - Tree *error_repeat = ts_tree_make_node( + Subtree *error_repeat = ts_subtree_make_node( &self->tree_pool, ts_builtin_sym_error_repeat, &children, @@ -1056,13 +1056,13 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) if (node_count_since_error > 0) { StackSliceArray pop = ts_stack_pop_count(self->stack, version, 1); assert(pop.size == 1); - assert(pop.contents[0].trees.size == 1); + assert(pop.contents[0].subtrees.size == 1); ts_stack_renumber_version(self->stack, pop.contents[0].version, version); - array_push(&pop.contents[0].trees, error_repeat); - error_repeat = ts_tree_make_node( + array_push(&pop.contents[0].subtrees, error_repeat); + error_repeat = ts_subtree_make_node( &self->tree_pool, ts_builtin_sym_error_repeat, - &pop.contents[0].trees, + &pop.contents[0].subtrees, 0, self->language ); @@ -1072,7 +1072,7 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) if (lookahead->has_external_tokens) { ts_stack_set_last_external_token( - self->stack, version, ts_tree_last_external_token(lookahead) + self->stack, version, ts_subtree_last_external_token(lookahead) ); } } @@ -1080,7 +1080,7 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead) static void parser__advance(Parser *self, StackVersion version, ReusableNode *reusable_node) { TSStateId state = ts_stack_state(self->stack, version); TableEntry table_entry; - Tree *lookahead = parser__get_lookahead(self, version, &state, reusable_node, &table_entry); + Subtree *lookahead = parser__get_lookahead(self, version, &state, reusable_node, &table_entry); for (;;) { StackVersion last_reduction_version = STACK_VERSION_NONE; @@ -1113,7 +1113,7 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re if (lookahead == reusable_node_tree(reusable_node)) { reusable_node_advance(reusable_node); } - ts_tree_release(&self->tree_pool, lookahead); + ts_subtree_release(&self->tree_pool, lookahead); return; } @@ -1133,7 +1133,7 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re case TSParseActionTypeAccept: { LOG("accept"); parser__accept(self, version, lookahead); - ts_tree_release(&self->tree_pool, lookahead); + ts_subtree_release(&self->tree_pool, lookahead); return; } @@ -1145,7 +1145,7 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re if (lookahead == reusable_node_tree(reusable_node)) { reusable_node_advance(reusable_node); } - ts_tree_release(&self->tree_pool, lookahead); + ts_subtree_release(&self->tree_pool, lookahead); return; } } @@ -1156,12 +1156,12 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re LOG_STACK(); } else if (state == ERROR_STATE) { parser__recover(self, version, lookahead); - ts_tree_release(&self->tree_pool, lookahead); + ts_subtree_release(&self->tree_pool, lookahead); return; } else if (!parser__breakdown_top_of_stack(self, version)) { LOG("detect_error"); ts_stack_pause(self->stack, version, lookahead->first_leaf.symbol); - ts_tree_release(&self->tree_pool, lookahead); + ts_subtree_release(&self->tree_pool, lookahead); return; } @@ -1260,7 +1260,7 @@ bool parser_init(Parser *self) { ts_lexer_init(&self->lexer); array_init(&self->reduce_actions); array_reserve(&self->reduce_actions, 4); - ts_tree_pool_init(&self->tree_pool); + ts_subtree_pool_init(&self->tree_pool); self->stack = ts_stack_new(&self->tree_pool); self->finished_tree = NULL; self->reusable_node = reusable_node_new(); @@ -1285,12 +1285,12 @@ void parser_destroy(Parser *self) { ts_stack_delete(self->stack); if (self->reduce_actions.contents) array_delete(&self->reduce_actions); - ts_tree_pool_delete(&self->tree_pool); + ts_subtree_pool_delete(&self->tree_pool); reusable_node_delete(&self->reusable_node); parser_set_language(self, NULL); } -Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_error) { +Subtree *parser_parse(Parser *self, TSInput input, Subtree *old_tree, bool halt_on_error) { parser__start(self, input, old_tree); StackVersion version = STACK_VERSION_NONE; @@ -1336,7 +1336,7 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_err reusable_node_delete(&reusable_node); ts_stack_clear(self->stack); parser__set_cached_token(self, 0, NULL, NULL); - ts_tree_balance(self->finished_tree, &self->tree_pool, self->language); + ts_subtree_balance(self->finished_tree, &self->tree_pool, self->language); LOG("done"); LOG_TREE(); diff --git a/src/runtime/parser.h b/src/runtime/parser.h index dab58dc9..ff776bc2 100644 --- a/src/runtime/parser.h +++ b/src/runtime/parser.h @@ -10,22 +10,22 @@ extern "C" { #include "runtime/lexer.h" #include "runtime/reusable_node.h" #include "runtime/reduce_action.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" typedef struct { - Tree *token; - Tree *last_external_token; + Subtree *token; + Subtree *last_external_token; uint32_t byte_index; } TokenCache; typedef struct { Lexer lexer; Stack *stack; - TreePool tree_pool; + SubtreePool tree_pool; const TSLanguage *language; ReduceActionSet reduce_actions; - Tree *finished_tree; - Tree scratch_tree; + Subtree *finished_tree; + Subtree scratch_tree; TokenCache token_cache; ReusableNode reusable_node; void *external_scanner_payload; @@ -36,7 +36,7 @@ typedef struct { bool parser_init(Parser *); void parser_destroy(Parser *); -Tree *parser_parse(Parser *, TSInput, Tree *, bool halt_on_error); +Subtree *parser_parse(Parser *, TSInput, Subtree *, bool halt_on_error); void parser_set_language(Parser *, const TSLanguage *); #ifdef __cplusplus diff --git a/src/runtime/reusable_node.h b/src/runtime/reusable_node.h index 71d56b68..5f1884d5 100644 --- a/src/runtime/reusable_node.h +++ b/src/runtime/reusable_node.h @@ -1,21 +1,21 @@ -#include "runtime/tree.h" +#include "runtime/subtree.h" typedef struct { - Tree *tree; + Subtree *tree; uint32_t child_index; uint32_t byte_offset; } StackEntry; typedef struct { Array(StackEntry) stack; - Tree *last_external_token; + Subtree *last_external_token; } ReusableNode; static inline ReusableNode reusable_node_new() { return (ReusableNode) {array_new(), NULL}; } -static inline void reusable_node_reset(ReusableNode *self, Tree *tree) { +static inline void reusable_node_reset(ReusableNode *self, Subtree *tree) { array_clear(&self->stack); array_push(&self->stack, ((StackEntry) { .tree = tree, @@ -24,7 +24,7 @@ static inline void reusable_node_reset(ReusableNode *self, Tree *tree) { })); } -static inline Tree *reusable_node_tree(ReusableNode *self) { +static inline Subtree *reusable_node_tree(ReusableNode *self) { return self->stack.size > 0 ? self->stack.contents[self->stack.size - 1].tree : NULL; @@ -46,12 +46,12 @@ static inline void reusable_node_assign(ReusableNode *self, const ReusableNode * static inline void reusable_node_advance(ReusableNode *self) { StackEntry last_entry = *array_back(&self->stack); - uint32_t byte_offset = last_entry.byte_offset + ts_tree_total_bytes(last_entry.tree); + uint32_t byte_offset = last_entry.byte_offset + ts_subtree_total_bytes(last_entry.tree); if (last_entry.tree->has_external_tokens) { - self->last_external_token = ts_tree_last_external_token(last_entry.tree); + self->last_external_token = ts_subtree_last_external_token(last_entry.tree); } - Tree *tree; + Subtree *tree; uint32_t next_index; do { StackEntry popped_entry = array_pop(&self->stack); diff --git a/src/runtime/stack.c b/src/runtime/stack.c index f7383846..c8af7a58 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -1,6 +1,6 @@ #include "runtime/alloc.h" #include "runtime/language.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/array.h" #include "runtime/stack.h" #include "runtime/length.h" @@ -21,7 +21,7 @@ typedef struct StackNode StackNode; typedef struct { StackNode *node; - Tree *tree; + Subtree *subtree; bool is_pending; } StackLink; @@ -38,8 +38,8 @@ struct StackNode { typedef struct { StackNode *node; - TreeArray trees; - uint32_t tree_count; + SubtreeArray subtrees; + uint32_t subtree_count; bool is_pending; } Iterator; @@ -58,7 +58,7 @@ typedef enum { typedef struct { StackNode *node; - Tree *last_external_token; + Subtree *last_external_token; StackSummary *summary; unsigned node_count_at_last_error; TSSymbol lookahead_when_paused; @@ -71,7 +71,7 @@ struct Stack { Array(Iterator) iterators; StackNodeArray node_pool; StackNode *base_node; - TreePool *tree_pool; + SubtreePool *subtree_pool; }; typedef unsigned StackAction; @@ -91,7 +91,7 @@ static void stack_node_retain(StackNode *self) { assert(self->ref_count != 0); } -static void stack_node_release(StackNode *self, StackNodeArray *pool, TreePool *tree_pool) { +static void stack_node_release(StackNode *self, StackNodeArray *pool, SubtreePool *subtree_pool) { recur: assert(self->ref_count != 0); self->ref_count--; @@ -100,10 +100,10 @@ recur: StackNode *first_predecessor = NULL; if (self->link_count > 0) { for (unsigned i = self->link_count - 1; i > 0; i--) { - if (self->links[i].tree) ts_tree_release(tree_pool, self->links[i].tree); - stack_node_release(self->links[i].node, pool, tree_pool); + if (self->links[i].subtree) ts_subtree_release(subtree_pool, self->links[i].subtree); + stack_node_release(self->links[i].node, pool, subtree_pool); } - if (self->links[0].tree) ts_tree_release(tree_pool, self->links[0].tree); + if (self->links[0].subtree) ts_subtree_release(subtree_pool, self->links[0].subtree); first_predecessor = self->links[0].node; } @@ -119,7 +119,7 @@ recur: } } -static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_pending, +static StackNode *stack_node_new(StackNode *previous_node, Subtree *subtree, bool is_pending, TSStateId state, StackNodeArray *pool) { StackNode *node = pool->size > 0 ? array_pop(pool) : @@ -130,7 +130,7 @@ static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_p node->link_count = 1; node->links[0] = (StackLink){ .node = previous_node, - .tree = tree, + .subtree = subtree, .is_pending = is_pending, }; @@ -139,11 +139,11 @@ static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_p node->dynamic_precedence = previous_node->dynamic_precedence; node->node_count = previous_node->node_count; - if (tree) { - node->error_cost += tree->error_cost; - node->position = length_add(node->position, ts_tree_total_size(tree)); - node->dynamic_precedence += tree->dynamic_precedence; - if (!tree->extra) node->node_count += tree->node_count; + if (subtree) { + node->error_cost += subtree->error_cost; + node->position = length_add(node->position, ts_subtree_total_size(subtree)); + node->dynamic_precedence += subtree->dynamic_precedence; + if (!subtree->extra) node->node_count += subtree->node_count; } } else { node->position = length_zero(); @@ -153,7 +153,7 @@ static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_p return node; } -static bool stack__tree_is_equivalent(const Tree *left, const Tree *right) { +static bool stack__subtree_is_equivalent(const Subtree *left, const Subtree *right) { return left == right || (left && @@ -164,7 +164,7 @@ static bool stack__tree_is_equivalent(const Tree *left, const Tree *right) { left->padding.bytes == right->padding.bytes && left->size.bytes == right->size.bytes && left->extra == right->extra && - ts_tree_external_token_state_eq(left, right)))); + ts_subtree_external_token_state_eq(left, right)))); } static void stack_node_add_link(StackNode *self, StackLink link) { @@ -172,7 +172,7 @@ static void stack_node_add_link(StackNode *self, StackLink link) { for (int i = 0; i < self->link_count; i++) { StackLink existing_link = self->links[i]; - if (stack__tree_is_equivalent(existing_link.tree, link.tree)) { + if (stack__subtree_is_equivalent(existing_link.subtree, link.subtree)) { if (existing_link.node == link.node) return; if (existing_link.node->state == link.node->state && existing_link.node->position.bytes == link.node->position.bytes) { @@ -187,24 +187,24 @@ static void stack_node_add_link(StackNode *self, StackLink link) { if (self->link_count == MAX_LINK_COUNT) return; stack_node_retain(link.node); - if (link.tree) ts_tree_retain(link.tree); + if (link.subtree) ts_subtree_retain(link.subtree); self->links[self->link_count++] = link; unsigned node_count = link.node->node_count; - if (link.tree) node_count += link.tree->node_count; + if (link.subtree) node_count += link.subtree->node_count; if (node_count > self->node_count) self->node_count = node_count; } -static void stack_head_delete(StackHead *self, StackNodeArray *pool, TreePool *tree_pool) { +static void stack_head_delete(StackHead *self, StackNodeArray *pool, SubtreePool *subtree_pool) { if (self->node) { if (self->last_external_token) { - ts_tree_release(tree_pool, self->last_external_token); + ts_subtree_release(subtree_pool, self->last_external_token); } if (self->summary) { array_delete(self->summary); ts_free(self->summary); } - stack_node_release(self->node, pool, tree_pool); + stack_node_release(self->node, pool, subtree_pool); } } @@ -219,44 +219,44 @@ static StackVersion ts_stack__add_version(Stack *self, StackVersion original_ver }; array_push(&self->heads, head); stack_node_retain(node); - if (head.last_external_token) ts_tree_retain(head.last_external_token); + if (head.last_external_token) ts_subtree_retain(head.last_external_token); return (StackVersion)(self->heads.size - 1); } static void ts_stack__add_slice(Stack *self, StackVersion original_version, - StackNode *node, TreeArray *trees) { + StackNode *node, SubtreeArray *subtrees) { for (uint32_t i = self->slices.size - 1; i + 1 > 0; i--) { StackVersion version = self->slices.contents[i].version; if (self->heads.contents[version].node == node) { - StackSlice slice = {*trees, version}; + StackSlice slice = {*subtrees, version}; array_insert(&self->slices, i + 1, slice); return; } } StackVersion version = ts_stack__add_version(self, original_version, node); - StackSlice slice = { *trees, version }; + StackSlice slice = { *subtrees, version }; array_push(&self->slices, slice); } inline StackSliceArray stack__iter(Stack *self, StackVersion version, StackCallback callback, void *payload, - int goal_tree_count) { + int goal_subtree_count) { array_clear(&self->slices); array_clear(&self->iterators); StackHead *head = array_get(&self->heads, version); Iterator iterator = { .node = head->node, - .trees = array_new(), - .tree_count = 0, + .subtrees = array_new(), + .subtree_count = 0, .is_pending = true, }; - bool include_trees = false; - if (goal_tree_count >= 0) { - include_trees = true; - array_reserve(&iterator.trees, goal_tree_count); + bool include_subtrees = false; + if (goal_subtree_count >= 0) { + include_subtrees = true; + array_reserve(&iterator.subtrees, goal_subtree_count); } array_push(&self->iterators, iterator); @@ -271,21 +271,21 @@ inline StackSliceArray stack__iter(Stack *self, StackVersion version, bool should_stop = action & StackActionStop || node->link_count == 0; if (should_pop) { - TreeArray trees = iterator->trees; + SubtreeArray subtrees = iterator->subtrees; if (!should_stop) - ts_tree_array_copy(trees, &trees); - ts_tree_array_reverse(&trees); + ts_subtree_array_copy(subtrees, &subtrees); + ts_subtree_array_reverse(&subtrees); ts_stack__add_slice( self, version, node, - &trees + &subtrees ); } if (should_stop) { if (!should_pop) - ts_tree_array_delete(self->tree_pool, &iterator->trees); + ts_subtree_array_delete(self->subtree_pool, &iterator->subtrees); array_erase(&self->iterators, i); i--, size--; continue; @@ -303,24 +303,24 @@ inline StackSliceArray stack__iter(Stack *self, StackVersion version, Iterator current_iterator = self->iterators.contents[i]; array_push(&self->iterators, current_iterator); next_iterator = array_back(&self->iterators); - ts_tree_array_copy(next_iterator->trees, &next_iterator->trees); + ts_subtree_array_copy(next_iterator->subtrees, &next_iterator->subtrees); } next_iterator->node = link.node; - if (link.tree) { - if (include_trees) { - array_push(&next_iterator->trees, link.tree); - ts_tree_retain(link.tree); + if (link.subtree) { + if (include_subtrees) { + array_push(&next_iterator->subtrees, link.subtree); + ts_subtree_retain(link.subtree); } - if (!link.tree->extra) { - next_iterator->tree_count++; + if (!link.subtree->extra) { + next_iterator->subtree_count++; if (!link.is_pending) { next_iterator->is_pending = false; } } } else { - next_iterator->tree_count++; + next_iterator->subtree_count++; next_iterator->is_pending = false; } } @@ -330,7 +330,7 @@ inline StackSliceArray stack__iter(Stack *self, StackVersion version, return self->slices; } -Stack *ts_stack_new(TreePool *tree_pool) { +Stack *ts_stack_new(SubtreePool *subtree_pool) { Stack *self = ts_calloc(1, sizeof(Stack)); array_init(&self->heads); @@ -342,7 +342,7 @@ Stack *ts_stack_new(TreePool *tree_pool) { array_reserve(&self->iterators, 4); array_reserve(&self->node_pool, MAX_NODE_POOL_SIZE); - self->tree_pool = tree_pool; + self->subtree_pool = subtree_pool; self->base_node = stack_node_new(NULL, NULL, false, 1, &self->node_pool); ts_stack_clear(self); @@ -354,9 +354,9 @@ void ts_stack_delete(Stack *self) { array_delete(&self->slices); if (self->iterators.contents) array_delete(&self->iterators); - stack_node_release(self->base_node, &self->node_pool, self->tree_pool); + stack_node_release(self->base_node, &self->node_pool, self->subtree_pool); for (uint32_t i = 0; i < self->heads.size; i++) { - stack_head_delete(&self->heads.contents[i], &self->node_pool, self->tree_pool); + stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool); } array_clear(&self->heads); if (self->node_pool.contents) { @@ -380,14 +380,14 @@ Length ts_stack_position(const Stack *self, StackVersion version) { return array_get(&self->heads, version)->node->position; } -Tree *ts_stack_last_external_token(const Stack *self, StackVersion version) { +Subtree *ts_stack_last_external_token(const Stack *self, StackVersion version) { return array_get(&self->heads, version)->last_external_token; } -void ts_stack_set_last_external_token(Stack *self, StackVersion version, Tree *token) { +void ts_stack_set_last_external_token(Stack *self, StackVersion version, Subtree *token) { StackHead *head = array_get(&self->heads, version); - if (token) ts_tree_retain(token); - if (head->last_external_token) ts_tree_release(self->tree_pool, head->last_external_token); + if (token) ts_subtree_retain(token); + if (head->last_external_token) ts_subtree_release(self->subtree_pool, head->last_external_token); head->last_external_token = token; } @@ -396,7 +396,7 @@ unsigned ts_stack_error_cost(const Stack *self, StackVersion version) { unsigned result = head->node->error_cost; if ( head->status == StackStatusPaused || - (head->node->state == ERROR_STATE && !head->node->links[0].tree)) { + (head->node->state == ERROR_STATE && !head->node->links[0].subtree)) { result += ERROR_COST_PER_RECOVERY; } return result; @@ -410,10 +410,10 @@ unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version return head->node->node_count - head->node_count_at_last_error; } -void ts_stack_push(Stack *self, StackVersion version, Tree *tree, bool pending, TSStateId state) { +void ts_stack_push(Stack *self, StackVersion version, Subtree *subtree, bool pending, TSStateId state) { StackHead *head = array_get(&self->heads, version); - StackNode *new_node = stack_node_new(head->node, tree, pending, state, &self->node_pool); - if (!tree) head->node_count_at_last_error = new_node->node_count; + StackNode *new_node = stack_node_new(head->node, subtree, pending, state, &self->node_pool); + if (!subtree) head->node_count_at_last_error = new_node->node_count; head->node = new_node; } @@ -422,7 +422,7 @@ inline StackAction iterate_callback(void *payload, const Iterator *iterator) { session->callback( session->payload, iterator->node->state, - iterator->tree_count + iterator->subtree_count ); return StackActionNone; } @@ -434,8 +434,8 @@ void ts_stack_iterate(Stack *self, StackVersion version, } inline StackAction pop_count_callback(void *payload, const Iterator *iterator) { - unsigned *goal_tree_count = payload; - if (iterator->tree_count == *goal_tree_count) { + unsigned *goal_subtree_count = payload; + if (iterator->subtree_count == *goal_subtree_count) { return StackActionPop | StackActionStop; } else { return StackActionNone; @@ -447,7 +447,7 @@ StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, uint32_t c } inline StackAction pop_pending_callback(void *payload, const Iterator *iterator) { - if (iterator->tree_count >= 1) { + if (iterator->subtree_count >= 1) { if (iterator->is_pending) { return StackActionPop | StackActionStop; } else { @@ -468,9 +468,9 @@ StackSliceArray ts_stack_pop_pending(Stack *self, StackVersion version) { } inline StackAction pop_error_callback(void *payload, const Iterator *iterator) { - if (iterator->trees.size > 0) { + if (iterator->subtrees.size > 0) { bool *found_error = payload; - if (!*found_error && iterator->trees.contents[0]->symbol == ts_builtin_sym_error) { + if (!*found_error && iterator->subtrees.contents[0]->symbol == ts_builtin_sym_error) { *found_error = true; return StackActionPop | StackActionStop; } else { @@ -481,21 +481,21 @@ inline StackAction pop_error_callback(void *payload, const Iterator *iterator) { } } -TreeArray ts_stack_pop_error(Stack *self, StackVersion version) { +SubtreeArray ts_stack_pop_error(Stack *self, StackVersion version) { StackNode *node = array_get(&self->heads, version)->node; for (unsigned i = 0; i < node->link_count; i++) { - if (node->links[i].tree && node->links[i].tree->symbol == ts_builtin_sym_error) { + if (node->links[i].subtree && node->links[i].subtree->symbol == ts_builtin_sym_error) { bool found_error = false; StackSliceArray pop = stack__iter(self, version, pop_error_callback, &found_error, 1); if (pop.size > 0) { assert(pop.size == 1); ts_stack_renumber_version(self, pop.contents[0].version, version); - return pop.contents[0].trees; + return pop.contents[0].subtrees; } break; } } - return (TreeArray){.size = 0}; + return (SubtreeArray){.size = 0}; } inline StackAction pop_all_callback(void *payload, const Iterator *iterator) { @@ -514,7 +514,7 @@ typedef struct { inline StackAction summarize_stack_callback(void *payload, const Iterator *iterator) { SummarizeStackSession *session = payload; TSStateId state = iterator->node->state; - unsigned depth = iterator->tree_count; + unsigned depth = iterator->subtree_count; if (depth > session->max_depth) return StackActionStop; for (unsigned i = session->summary->size - 1; i + 1 > 0; i--) { StackSummaryEntry entry = session->summary->contents[i]; @@ -548,7 +548,7 @@ int ts_stack_dynamic_precedence(Stack *self, StackVersion version) { } void ts_stack_remove_version(Stack *self, StackVersion version) { - stack_head_delete(array_get(&self->heads, version), &self->node_pool, self->tree_pool); + stack_head_delete(array_get(&self->heads, version), &self->node_pool, self->subtree_pool); array_erase(&self->heads, version); } @@ -561,7 +561,7 @@ void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) { source_head->summary = target_head->summary; target_head->summary = NULL; } - stack_head_delete(target_head, &self->node_pool, self->tree_pool); + stack_head_delete(target_head, &self->node_pool, self->subtree_pool); *target_head = *source_head; array_erase(&self->heads, v1); } @@ -577,7 +577,7 @@ StackVersion ts_stack_copy_version(Stack *self, StackVersion version) { array_push(&self->heads, self->heads.contents[version]); StackHead *head = array_back(&self->heads); stack_node_retain(head->node); - if (head->last_external_token) ts_tree_retain(head->last_external_token); + if (head->last_external_token) ts_subtree_retain(head->last_external_token); head->summary = NULL; return self->heads.size - 1; } @@ -605,7 +605,7 @@ bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version head1->node->state == head2->node->state && head1->node->position.bytes == head2->node->position.bytes && head1->node->error_cost == head2->node->error_cost && - ts_tree_external_token_state_eq(head1->last_external_token, head2->last_external_token); + ts_subtree_external_token_state_eq(head1->last_external_token, head2->last_external_token); } void ts_stack_halt(Stack *self, StackVersion version) { @@ -643,7 +643,7 @@ TSSymbol ts_stack_resume(Stack *self, StackVersion version) { void ts_stack_clear(Stack *self) { stack_node_retain(self->base_node); for (uint32_t i = 0; i < self->heads.size; i++) { - stack_head_delete(&self->heads.contents[i], &self->node_pool, self->tree_pool); + stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool); } array_clear(&self->heads); array_push(&self->heads, ((StackHead){ @@ -715,8 +715,8 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) fprintf(f, "node_%p [", node); if (node->state == ERROR_STATE) fprintf(f, "label=\"?\""); - else if (node->link_count == 1 && node->links[0].tree && - node->links[0].tree->extra) + else if (node->link_count == 1 && node->links[0].subtree && + node->links[0].subtree->extra) fprintf(f, "shape=point margin=0 label=\"\""); else fprintf(f, "label=\"%d\"", node->state); @@ -736,24 +736,24 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) fprintf(f, "node_%p -> node_%p [", node, link.node); if (link.is_pending) fprintf(f, "style=dashed "); - if (link.tree && link.tree->extra) + if (link.subtree && link.subtree->extra) fprintf(f, "fontcolor=gray "); - if (!link.tree) { + if (!link.subtree) { fprintf(f, "color=red"); } else { fprintf(f, "label=\""); - if (link.tree->visible && !link.tree->named) fprintf(f, "'"); - const char *name = ts_language_symbol_name(language, link.tree->symbol); + if (link.subtree->visible && !link.subtree->named) fprintf(f, "'"); + const char *name = ts_language_symbol_name(language, link.subtree->symbol); for (const char *c = name; *c; c++) { if (*c == '\"' || *c == '\\') fprintf(f, "\\"); fprintf(f, "%c", *c); } - if (link.tree->visible && !link.tree->named) fprintf(f, "'"); + if (link.subtree->visible && !link.subtree->named) fprintf(f, "'"); fprintf(f, "\""); fprintf(f, "labeltooltip=\"error_cost: %u\ndynamic_precedence: %u\"", - link.tree->error_cost, - link.tree->dynamic_precedence); + link.subtree->error_cost, + link.subtree->dynamic_precedence); } fprintf(f, "];\n"); diff --git a/src/runtime/stack.h b/src/runtime/stack.h index 4a552323..88c06bab 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -6,7 +6,7 @@ extern "C" { #endif #include "runtime/array.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/error_costs.h" #include @@ -16,7 +16,7 @@ typedef unsigned StackVersion; #define STACK_VERSION_NONE ((StackVersion)-1) typedef struct { - TreeArray trees; + SubtreeArray subtrees; StackVersion version; } StackSlice; typedef Array(StackSlice) StackSliceArray; @@ -29,7 +29,7 @@ typedef struct { typedef Array(StackSummaryEntry) StackSummary; // Create a stack. -Stack *ts_stack_new(TreePool *); +Stack *ts_stack_new(SubtreePool *); // Release the memory reserved for a given stack. void ts_stack_delete(Stack *); @@ -42,10 +42,10 @@ uint32_t ts_stack_version_count(const Stack *); TSStateId ts_stack_state(const Stack *, StackVersion); // Get the last external token associated with a given version of the stack. -Tree *ts_stack_last_external_token(const Stack *, StackVersion); +Subtree *ts_stack_last_external_token(const Stack *, StackVersion); // Set the last external token associated with a given version of the stack. -void ts_stack_set_last_external_token(Stack *, StackVersion, Tree *); +void ts_stack_set_last_external_token(Stack *, StackVersion, Subtree *); // Get the position of the given version of the stack within the document. Length ts_stack_position(const Stack *, StackVersion); @@ -55,7 +55,7 @@ Length ts_stack_position(const Stack *, StackVersion); // This transfers ownership of the tree to the Stack. Callers that // need to retain ownership of the tree for their own purposes should // first retain the tree. -void ts_stack_push(Stack *, StackVersion, Tree *, bool, TSStateId); +void ts_stack_push(Stack *, StackVersion, Subtree *, bool, TSStateId); // Pop the given number of entries from the given version of the stack. This // operation can increase the number of stack versions by revealing multiple @@ -65,7 +65,7 @@ void ts_stack_push(Stack *, StackVersion, Tree *, bool, TSStateId); StackSliceArray ts_stack_pop_count(Stack *, StackVersion, uint32_t count); // Remove an error at the top of the given version of the stack. -TreeArray ts_stack_pop_error(Stack *, StackVersion); +SubtreeArray ts_stack_pop_error(Stack *, StackVersion); // Remove any pending trees from the top of the given version of the stack. StackSliceArray ts_stack_pop_pending(Stack *, StackVersion); diff --git a/src/runtime/tree.c b/src/runtime/subtree.c similarity index 70% rename from src/runtime/tree.c rename to src/runtime/subtree.c index 602392f4..1808ee6d 100644 --- a/src/runtime/tree.c +++ b/src/runtime/subtree.c @@ -6,7 +6,7 @@ #include #include "runtime/alloc.h" #include "runtime/atomic.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/length.h" #include "runtime/language.h" #include "runtime/error_costs.h" @@ -51,15 +51,15 @@ bool ts_external_token_state_eq(const TSExternalTokenState *a, const TSExternalT memcmp(ts_external_token_state_data(a), ts_external_token_state_data(b), a->length) == 0); } -// TreeArray +// SubtreeArray -bool ts_tree_array_copy(TreeArray self, TreeArray *dest) { - Tree **contents = NULL; +bool ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest) { + Subtree **contents = NULL; if (self.capacity > 0) { - contents = ts_calloc(self.capacity, sizeof(Tree *)); - memcpy(contents, self.contents, self.size * sizeof(Tree *)); + contents = ts_calloc(self.capacity, sizeof(Subtree *)); + memcpy(contents, self.contents, self.size * sizeof(Subtree *)); for (uint32_t i = 0; i < self.size; i++) { - ts_tree_retain(contents[i]); + ts_subtree_retain(contents[i]); } } @@ -69,47 +69,47 @@ bool ts_tree_array_copy(TreeArray self, TreeArray *dest) { return true; } -void ts_tree_array_delete(TreePool *pool, TreeArray *self) { +void ts_subtree_array_delete(SubtreePool *pool, SubtreeArray *self) { for (uint32_t i = 0; i < self->size; i++) { - ts_tree_release(pool, self->contents[i]); + ts_subtree_release(pool, self->contents[i]); } array_delete(self); } -TreeArray ts_tree_array_remove_trailing_extras(TreeArray *self) { - TreeArray result = array_new(); +SubtreeArray ts_subtree_array_remove_trailing_extras(SubtreeArray *self) { + SubtreeArray result = array_new(); uint32_t i = self->size - 1; for (; i + 1 > 0; i--) { - Tree *child = self->contents[i]; + Subtree *child = self->contents[i]; if (!child->extra) break; array_push(&result, child); } self->size = i + 1; - ts_tree_array_reverse(&result); + ts_subtree_array_reverse(&result); return result; } -void ts_tree_array_reverse(TreeArray *self) { +void ts_subtree_array_reverse(SubtreeArray *self) { for (uint32_t i = 0, limit = self->size / 2; i < limit; i++) { size_t reverse_index = self->size - 1 - i; - Tree *swap = self->contents[i]; + Subtree *swap = self->contents[i]; self->contents[i] = self->contents[reverse_index]; self->contents[reverse_index] = swap; } } -// TreePool +// SubtreePool static const uint32_t MAX_TREE_POOL_SIZE = 1024; -void ts_tree_pool_init(TreePool *self) { +void ts_subtree_pool_init(SubtreePool *self) { array_init(&self->free_trees); array_init(&self->tree_stack); } -void ts_tree_pool_delete(TreePool *self) { +void ts_subtree_pool_delete(SubtreePool *self) { if (self->free_trees.contents) { for (unsigned i = 0; i < self->free_trees.size; i++) { ts_free(self->free_trees.contents[i]); @@ -119,15 +119,15 @@ void ts_tree_pool_delete(TreePool *self) { if (self->tree_stack.contents) array_delete(&self->tree_stack); } -Tree *ts_tree_pool_allocate(TreePool *self) { +Subtree *ts_subtree_pool_allocate(SubtreePool *self) { if (self->free_trees.size > 0) { return array_pop(&self->free_trees); } else { - return ts_malloc(sizeof(Tree)); + return ts_malloc(sizeof(Subtree)); } } -void ts_tree_pool_free(TreePool *self, Tree *tree) { +void ts_subtree_pool_free(SubtreePool *self, Subtree *tree) { if (self->free_trees.size < MAX_TREE_POOL_SIZE) { array_push(&self->free_trees, tree); } else { @@ -135,12 +135,13 @@ void ts_tree_pool_free(TreePool *self, Tree *tree) { } } -// Tree +// Subtree -Tree *ts_tree_make_leaf(TreePool *pool, TSSymbol symbol, Length padding, Length size, const TSLanguage *language) { +Subtree *ts_subtree_make_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, Length size, + const TSLanguage *language) { TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); - Tree *result = ts_tree_pool_allocate(pool); - *result = (Tree){ + Subtree *result = ts_subtree_pool_allocate(pool); + *result = (Subtree){ .ref_count = 1, .symbol = symbol, .size = size, @@ -161,50 +162,51 @@ Tree *ts_tree_make_leaf(TreePool *pool, TSSymbol symbol, Length padding, Length return result; } -Tree *ts_tree_make_error(TreePool *pool, Length size, Length padding, int32_t lookahead_char, - const TSLanguage *language) { - Tree *result = ts_tree_make_leaf(pool, ts_builtin_sym_error, padding, size, language); +Subtree *ts_subtree_make_error(SubtreePool *pool, Length size, Length padding, + int32_t lookahead_char, const TSLanguage *language) { + Subtree *result = ts_subtree_make_leaf(pool, ts_builtin_sym_error, padding, size, language); result->fragile_left = true; result->fragile_right = true; result->lookahead_char = lookahead_char; return result; } -Tree *ts_tree_make_copy(TreePool *pool, Tree *self) { - Tree *result = ts_tree_pool_allocate(pool); +Subtree *ts_subtree_make_copy(SubtreePool *pool, Subtree *self) { + Subtree *result = ts_subtree_pool_allocate(pool); *result = *self; if (result->children.size > 0) { - ts_tree_array_copy(self->children, &result->children); + ts_subtree_array_copy(self->children, &result->children); } result->ref_count = 1; return result; } -static Tree *ts_tree_make_mut(TreePool *pool, Tree *self) { +static Subtree *ts_subtree_make_mut(SubtreePool *pool, Subtree *self) { if (self->ref_count == 1) { return self; } else { - Tree *result = ts_tree_make_copy(pool, self); - ts_tree_release(pool, self); + Subtree *result = ts_subtree_make_copy(pool, self); + ts_subtree_release(pool, self); return result; } } -static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *language, TreeArray *stack) { +static void ts_subtree__compress(Subtree *self, unsigned count, const TSLanguage *language, + SubtreeArray *stack) { unsigned initial_stack_size = stack->size; - Tree *tree = self; + Subtree *tree = self; for (unsigned i = 0; i < count; i++) { if (tree->ref_count > 1 || tree->children.size != 2) break; - Tree *child = tree->children.contents[0]; + Subtree *child = tree->children.contents[0]; if ( child->ref_count > 1 || child->children.size != 2 || child->symbol != tree->symbol ) break; - Tree *grandchild = child->children.contents[0]; + Subtree *grandchild = child->children.contents[0]; if ( grandchild->ref_count > 1 || grandchild->children.size != 2 || @@ -221,19 +223,19 @@ static void ts_tree__compress(Tree *self, unsigned count, const TSLanguage *lang while (stack->size > initial_stack_size) { tree = array_pop(stack); assert(tree); - Tree *child = tree->children.contents[0]; - Tree *grandchild = child->children.contents[1]; - ts_tree_set_children(grandchild, &grandchild->children, language); - ts_tree_set_children(child, &child->children, language); - ts_tree_set_children(tree, &tree->children, language); + Subtree *child = tree->children.contents[0]; + Subtree *grandchild = child->children.contents[1]; + ts_subtree_set_children(grandchild, &grandchild->children, language); + ts_subtree_set_children(child, &child->children, language); + ts_subtree_set_children(tree, &tree->children, language); } } -void ts_tree_balance(Tree *self, TreePool *pool, const TSLanguage *language) { +void ts_subtree_balance(Subtree *self, SubtreePool *pool, const TSLanguage *language) { array_clear(&pool->tree_stack); array_push(&pool->tree_stack, self); while (pool->tree_stack.size > 0) { - Tree *tree = array_pop(&pool->tree_stack); + Subtree *tree = array_pop(&pool->tree_stack); assert(tree); if (tree->repeat_depth > 0) { @@ -243,14 +245,14 @@ void ts_tree_balance(Tree *self, TreePool *pool, const TSLanguage *language) { tree->children.contents[1]->repeat_depth ); for (unsigned i = n / 2; i > 0; i /= 2) { - ts_tree__compress(tree, i, language, &pool->tree_stack); + ts_subtree__compress(tree, i, language, &pool->tree_stack); n -= i; } } } for (uint32_t i = 0; i < tree->children.size; i++) { - Tree *child = tree->children.contents[i]; + Subtree *child = tree->children.contents[i]; if (child->ref_count == 1) { array_push(&pool->tree_stack, child); } @@ -258,7 +260,7 @@ void ts_tree_balance(Tree *self, TreePool *pool, const TSLanguage *language) { } } -void ts_tree_set_children(Tree *self, TreeArray *children, const TSLanguage *language) { +void ts_subtree_set_children(Subtree *self, SubtreeArray *children, const TSLanguage *language) { if (self->children.size > 0 && children->contents != self->children.contents) { array_delete(&self->children); } @@ -276,16 +278,16 @@ void ts_tree_set_children(Tree *self, TreeArray *children, const TSLanguage *lan const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); for (uint32_t i = 0; i < self->children.size; i++) { - Tree *child = self->children.contents[i]; + Subtree *child = self->children.contents[i]; if (i == 0) { self->padding = child->padding; self->size = child->size; self->bytes_scanned = child->bytes_scanned; } else { - uint32_t bytes_scanned = ts_tree_total_bytes(self) + child->bytes_scanned; + uint32_t bytes_scanned = ts_subtree_total_bytes(self) + child->bytes_scanned; if (bytes_scanned > self->bytes_scanned) self->bytes_scanned = bytes_scanned; - self->size = length_add(self->size, ts_tree_total_size(child)); + self->size = length_add(self->size, ts_subtree_total_size(child)); } if (child->symbol != ts_builtin_sym_error_repeat) { @@ -322,7 +324,7 @@ void ts_tree_set_children(Tree *self, TreeArray *children, const TSLanguage *lan ERROR_COST_PER_SKIPPED_CHAR * self->size.bytes + ERROR_COST_PER_SKIPPED_LINE * self->size.extent.row; for (uint32_t i = 0; i < self->children.size; i++) { - Tree *child = self->children.contents[i]; + Subtree *child = self->children.contents[i]; if (child->extra) continue; if (child->symbol == ts_builtin_sym_error && child->children.size == 0) continue; if (child->visible) { @@ -334,8 +336,8 @@ void ts_tree_set_children(Tree *self, TreeArray *children, const TSLanguage *lan } if (self->children.size > 0) { - Tree *first_child = self->children.contents[0]; - Tree *last_child = self->children.contents[self->children.size - 1]; + Subtree *first_child = self->children.contents[0]; + Subtree *last_child = self->children.contents[self->children.size - 1]; self->first_leaf = first_child->first_leaf; if (first_child->fragile_left) self->fragile_left = true; if (last_child->fragile_right) self->fragile_right = true; @@ -354,40 +356,42 @@ void ts_tree_set_children(Tree *self, TreeArray *children, const TSLanguage *lan } } -Tree *ts_tree_make_node(TreePool *pool, TSSymbol symbol, TreeArray *children, +Subtree *ts_subtree_make_node(SubtreePool *pool, TSSymbol symbol, SubtreeArray *children, unsigned alias_sequence_id, const TSLanguage *language) { - Tree *result = ts_tree_make_leaf(pool, symbol, length_zero(), length_zero(), language); + Subtree *result = ts_subtree_make_leaf(pool, symbol, length_zero(), length_zero(), language); result->alias_sequence_id = alias_sequence_id; if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { result->fragile_left = true; result->fragile_right = true; } - ts_tree_set_children(result, children, language); + ts_subtree_set_children(result, children, language); return result; } -Tree *ts_tree_make_error_node(TreePool *pool, TreeArray *children, const TSLanguage *language) { - return ts_tree_make_node(pool, ts_builtin_sym_error, children, 0, language); +Subtree *ts_subtree_make_error_node(SubtreePool *pool, SubtreeArray *children, + const TSLanguage *language) { + return ts_subtree_make_node(pool, ts_builtin_sym_error, children, 0, language); } -Tree *ts_tree_make_missing_leaf(TreePool *pool, TSSymbol symbol, const TSLanguage *language) { - Tree *result = ts_tree_make_leaf(pool, symbol, length_zero(), length_zero(), language); +Subtree *ts_subtree_make_missing_leaf(SubtreePool *pool, TSSymbol symbol, + const TSLanguage *language) { + Subtree *result = ts_subtree_make_leaf(pool, symbol, length_zero(), length_zero(), language); result->is_missing = true; result->error_cost = ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY; return result; } -void ts_tree_retain(Tree *self) { +void ts_subtree_retain(Subtree *self) { assert(self->ref_count > 0); atomic_inc(&self->ref_count); assert(self->ref_count != 0); } -void ts_tree_release(TreePool *pool, Tree *self) { +void ts_subtree_release(SubtreePool *pool, Subtree *self) { array_clear(&pool->tree_stack); array_push(&pool->tree_stack, self); while (pool->tree_stack.size > 0) { - Tree *tree = array_pop(&pool->tree_stack); + Subtree *tree = array_pop(&pool->tree_stack); assert(tree->ref_count > 0); if (atomic_dec(&tree->ref_count) == 0) { if (tree->children.size > 0) { @@ -398,12 +402,12 @@ void ts_tree_release(TreePool *pool, Tree *self) { } else if (tree->has_external_tokens) { ts_external_token_state_delete(&tree->external_token_state); } - ts_tree_pool_free(pool, tree); + ts_subtree_pool_free(pool, tree); } } } -bool ts_tree_eq(const Tree *self, const Tree *other) { +bool ts_subtree_eq(const Subtree *self, const Subtree *other) { if (self) { if (!other) return false; } else { @@ -421,14 +425,14 @@ bool ts_tree_eq(const Tree *self, const Tree *other) { if (self->named_child_count != other->named_child_count) return false; for (uint32_t i = 0; i < self->children.size; i++) { - if (!ts_tree_eq(self->children.contents[i], other->children.contents[i])) { + if (!ts_subtree_eq(self->children.contents[i], other->children.contents[i])) { return false; } } return true; } -int ts_tree_compare(const Tree *left, const Tree *right) { +int ts_subtree_compare(const Subtree *left, const Subtree *right) { if (left->symbol < right->symbol) return -1; if (right->symbol < left->symbol) @@ -438,9 +442,9 @@ int ts_tree_compare(const Tree *left, const Tree *right) { if (right->children.size < left->children.size) return 1; for (uint32_t i = 0; i < left->children.size; i++) { - Tree *left_child = left->children.contents[i]; - Tree *right_child = right->children.contents[i]; - switch (ts_tree_compare(left_child, right_child)) { + Subtree *left_child = left->children.contents[i]; + Subtree *right_child = right->children.contents[i]; + switch (ts_subtree_compare(left_child, right_child)) { case -1: return -1; case 1: @@ -452,30 +456,31 @@ int ts_tree_compare(const Tree *left, const Tree *right) { return 0; } -Tree *ts_tree_invalidate_lookahead(Tree *self, uint32_t edit_byte_offset, TreePool *pool) { +Subtree *ts_subtree_invalidate_lookahead(Subtree *self, uint32_t edit_byte_offset, + SubtreePool *pool) { if (edit_byte_offset >= self->bytes_scanned) return self; - Tree *result = ts_tree_make_mut(pool, self); + Subtree *result = ts_subtree_make_mut(pool, self); result->has_changes = true; if (result->children.size > 0) { uint32_t child_start_byte = 0; for (uint32_t i = 0; i < result->children.size; i++) { - Tree **child = &result->children.contents[i]; + Subtree **child = &result->children.contents[i]; if (child_start_byte > edit_byte_offset) break; - *child = ts_tree_invalidate_lookahead(*child, edit_byte_offset - child_start_byte, pool); - child_start_byte += ts_tree_total_bytes(*child); + *child = ts_subtree_invalidate_lookahead(*child, edit_byte_offset - child_start_byte, pool); + child_start_byte += ts_subtree_total_bytes(*child); } } return result; } -Tree *ts_tree__edit(Tree *self, Edit edit, TreePool *pool) { +Subtree *ts_subtree__edit(Subtree *self, Edit edit, SubtreePool *pool) { Length new_end = length_add(edit.start, edit.added); Length old_end = length_add(edit.start, edit.removed); - Tree *result = ts_tree_make_mut(pool, self); + Subtree *result = ts_subtree_make_mut(pool, self); result->has_changes = true; if (old_end.bytes <= result->padding.bytes) { @@ -486,14 +491,14 @@ Tree *ts_tree__edit(Tree *self, Edit edit, TreePool *pool) { } else if (edit.start.bytes == result->padding.bytes && edit.removed.bytes == 0) { result->padding = length_add(result->padding, edit.added); } else { - Length new_total_size = length_add(new_end, length_sub(ts_tree_total_size(result), old_end)); + Length new_total_size = length_add(new_end, length_sub(ts_subtree_total_size(result), old_end)); result->size = length_sub(new_total_size, result->padding); } Length child_left, child_right = length_zero(); for (uint32_t i = 0; i < result->children.size; i++) { - Tree **child = &result->children.contents[i]; - Length child_size = ts_tree_total_size(*child); + Subtree **child = &result->children.contents[i]; + Length child_size = ts_subtree_total_size(*child); child_left = child_right; child_right = length_add(child_left, child_size); @@ -519,28 +524,28 @@ Tree *ts_tree__edit(Tree *self, Edit edit, TreePool *pool) { edit.added = length_zero(); edit.removed = length_sub(edit.removed, child_edit.removed); - *child = ts_tree__edit(*child, child_edit, pool); + *child = ts_subtree__edit(*child, child_edit, pool); } else if (child_left.bytes <= edit.start.bytes) { - *child = ts_tree_invalidate_lookahead(*child, edit.start.bytes - child_left.bytes, pool); + *child = ts_subtree_invalidate_lookahead(*child, edit.start.bytes - child_left.bytes, pool); } } return result; } -Tree *ts_tree_edit(Tree *self, const TSInputEdit *edit, TreePool *pool) { - return ts_tree__edit(self, (Edit) { +Subtree *ts_subtree_edit(Subtree *self, const TSInputEdit *edit, SubtreePool *pool) { + return ts_subtree__edit(self, (Edit) { .start = {edit->start_byte, edit->start_point}, .added = {edit->bytes_added, edit->extent_added}, .removed = {edit->bytes_removed, edit->extent_removed}, }, pool); } -Tree *ts_tree_last_external_token(Tree *tree) { +Subtree *ts_subtree_last_external_token(Subtree *tree) { if (!tree->has_external_tokens) return NULL; while (tree->children.size > 0) { for (uint32_t i = tree->children.size - 1; i + 1 > 0; i--) { - Tree *child = tree->children.contents[i]; + Subtree *child = tree->children.contents[i]; if (child->has_external_tokens) { tree = child; break; @@ -550,7 +555,7 @@ Tree *ts_tree_last_external_token(Tree *tree) { return tree; } -static size_t ts_tree__write_char_to_string(char *s, size_t n, int32_t c) { +static size_t ts_subtree__write_char_to_string(char *s, size_t n, int32_t c) { if (c == 0) return snprintf(s, n, "EOF"); if (c == -1) @@ -567,7 +572,7 @@ static size_t ts_tree__write_char_to_string(char *s, size_t n, int32_t c) { return snprintf(s, n, "%d", c); } -static size_t ts_tree__write_to_string(const Tree *self, char *string, size_t limit, +static size_t ts_subtree__write_to_string(const Subtree *self, char *string, size_t limit, const TSLanguage *language, bool is_root, bool include_all, TSSymbol alias_symbol, bool alias_is_named) { @@ -589,7 +594,7 @@ static size_t ts_tree__write_to_string(const Tree *self, char *string, size_t li if (visible) { if (self->symbol == ts_builtin_sym_error && self->children.size == 0 && self->size.bytes > 0) { cursor += snprintf(*writer, limit, "(UNEXPECTED "); - cursor += ts_tree__write_char_to_string(*writer, limit, self->lookahead_char); + cursor += ts_subtree__write_char_to_string(*writer, limit, self->lookahead_char); } else if (self->is_missing) { cursor += snprintf(*writer, limit, "(MISSING"); } else { @@ -602,16 +607,16 @@ static size_t ts_tree__write_to_string(const Tree *self, char *string, size_t li const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); uint32_t structural_child_index = 0; for (uint32_t i = 0; i < self->children.size; i++) { - Tree *child = self->children.contents[i]; + Subtree *child = self->children.contents[i]; if (child->extra) { - cursor += ts_tree__write_to_string( + cursor += ts_subtree__write_to_string( child, *writer, limit, language, false, include_all, 0, false ); } else { TSSymbol alias_symbol = alias_sequence ? alias_sequence[structural_child_index] : 0; - cursor += ts_tree__write_to_string( + cursor += ts_subtree__write_to_string( child, *writer, limit, language, false, include_all, alias_symbol, @@ -626,19 +631,19 @@ static size_t ts_tree__write_to_string(const Tree *self, char *string, size_t li return cursor - string; } -char *ts_tree_string(const Tree *self, const TSLanguage *language, bool include_all) { +char *ts_subtree_string(const Subtree *self, const TSLanguage *language, bool include_all) { char scratch_string[1]; - size_t size = ts_tree__write_to_string( + size_t size = ts_subtree__write_to_string( self, scratch_string, 0, language, true, include_all, 0, false ) + 1; char *result = ts_malloc(size * sizeof(char)); - ts_tree__write_to_string(self, result, size, language, true, include_all, 0, false); + ts_subtree__write_to_string(self, result, size, language, true, include_all, 0, false); return result; } -void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset, +void ts_subtree__print_dot_graph(const Subtree *self, uint32_t byte_offset, const TSLanguage *language, TSSymbol alias_symbol, FILE *f) { TSSymbol symbol = alias_symbol ? alias_symbol : self->symbol; fprintf(f, "tree_%p [label=\"%s\"", self, ts_language_symbol_name(language, symbol)); @@ -656,7 +661,7 @@ void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset, "repeat-depth:%u\n" "bytes-scanned:%u\"]\n", self, - byte_offset, byte_offset + ts_tree_total_bytes(self), + byte_offset, byte_offset + ts_subtree_total_bytes(self), self->parse_state, self->error_cost, self->repeat_depth, @@ -666,29 +671,29 @@ void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset, const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); uint32_t structural_child_index = 0; for (uint32_t i = 0; i < self->children.size; i++) { - const Tree *child = self->children.contents[i]; + const Subtree *child = self->children.contents[i]; if (child->extra) { - ts_tree__print_dot_graph(child, byte_offset, language, 0, f); + ts_subtree__print_dot_graph(child, byte_offset, language, 0, f); } else { TSSymbol alias_symbol = alias_sequence ? alias_sequence[structural_child_index] : 0; - ts_tree__print_dot_graph(child, byte_offset, language, alias_symbol, f); + ts_subtree__print_dot_graph(child, byte_offset, language, alias_symbol, f); structural_child_index++; } fprintf(f, "tree_%p -> tree_%p [tooltip=%u]\n", self, child, i); - byte_offset += ts_tree_total_bytes(child); + byte_offset += ts_subtree_total_bytes(child); } } -void ts_tree_print_dot_graph(const Tree *self, const TSLanguage *language, FILE *f) { +void ts_subtree_print_dot_graph(const Subtree *self, const TSLanguage *language, FILE *f) { fprintf(f, "digraph tree {\n"); fprintf(f, "edge [arrowhead=none]\n"); - ts_tree__print_dot_graph(self, 0, language, 0, f); + ts_subtree__print_dot_graph(self, 0, language, 0, f); fprintf(f, "}\n"); } static const TSExternalTokenState empty_state = {.length = 0, .short_data = {0}}; -bool ts_tree_external_token_state_eq(const Tree *self, const Tree *other) { +bool ts_subtree_external_token_state_eq(const Subtree *self, const Subtree *other) { const TSExternalTokenState *state1 = &empty_state; const TSExternalTokenState *state2 = &empty_state; if (self && self->has_external_tokens) state1 = &self->external_token_state; diff --git a/src/runtime/subtree.h b/src/runtime/subtree.h new file mode 100644 index 00000000..5168ac38 --- /dev/null +++ b/src/runtime/subtree.h @@ -0,0 +1,125 @@ +#ifndef RUNTIME_TREE_H_ +#define RUNTIME_TREE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include "tree_sitter/parser.h" +#include "tree_sitter/runtime.h" +#include "runtime/length.h" +#include "runtime/array.h" +#include + +extern TSStateId TS_TREE_STATE_NONE; + +typedef struct { + union { + char *long_data; + char short_data[sizeof(char *) + sizeof(uint32_t)]; + }; + uint32_t length; +} TSExternalTokenState; + +typedef struct Subtree Subtree; + +typedef Array(Subtree *) SubtreeArray; + +struct Subtree { + Length padding; + Length size; + volatile uint32_t ref_count; + uint32_t bytes_scanned; + uint32_t error_cost; + uint32_t node_count; + uint32_t repeat_depth; + uint32_t child_count; + int32_t dynamic_precedence; + + bool visible : 1; + bool named : 1; + bool extra : 1; + bool fragile_left : 1; + bool fragile_right : 1; + bool has_changes : 1; + bool has_external_tokens : 1; + bool is_missing : 1; + TSSymbol symbol; + TSStateId parse_state; + struct { + TSSymbol symbol; + TSLexMode lex_mode; + } first_leaf; + + union { + struct { + SubtreeArray children; + uint32_t visible_child_count; + uint32_t named_child_count; + uint16_t alias_sequence_id; + }; + struct { + uint32_t _2; + TSExternalTokenState external_token_state; + }; + struct { + uint32_t _1; + int32_t lookahead_char; + }; + }; +}; + +typedef struct { + SubtreeArray free_trees; + SubtreeArray tree_stack; +} SubtreePool; + +void ts_external_token_state_init(TSExternalTokenState *, const char *, unsigned); +const char *ts_external_token_state_data(const TSExternalTokenState *); + +bool ts_subtree_array_copy(SubtreeArray, SubtreeArray *); +void ts_subtree_array_delete(SubtreePool *, SubtreeArray *); +SubtreeArray ts_subtree_array_remove_trailing_extras(SubtreeArray *); +void ts_subtree_array_reverse(SubtreeArray *); + +void ts_subtree_pool_init(SubtreePool *); +void ts_subtree_pool_delete(SubtreePool *); +Subtree *ts_subtree_pool_allocate(SubtreePool *); +void ts_subtree_pool_free(SubtreePool *, Subtree *); + +Subtree *ts_subtree_make_leaf(SubtreePool *, TSSymbol, Length, Length, const TSLanguage *); +Subtree *ts_subtree_make_node(SubtreePool *, TSSymbol, SubtreeArray *, unsigned, const TSLanguage *); +Subtree *ts_subtree_make_copy(SubtreePool *, Subtree *child); +Subtree *ts_subtree_make_error_node(SubtreePool *, SubtreeArray *, const TSLanguage *); +Subtree *ts_subtree_make_error(SubtreePool *, Length, Length, int32_t, const TSLanguage *); +Subtree *ts_subtree_make_missing_leaf(SubtreePool *, TSSymbol, const TSLanguage *); +void ts_subtree_retain(Subtree *tree); +void ts_subtree_release(SubtreePool *, Subtree *tree); +bool ts_subtree_eq(const Subtree *tree1, const Subtree *tree2); +int ts_subtree_compare(const Subtree *tree1, const Subtree *tree2); +void ts_subtree_set_children(Subtree *, SubtreeArray *, const TSLanguage *); +void ts_subtree_balance(Subtree *, SubtreePool *, const TSLanguage *); +Subtree *ts_subtree_edit(Subtree *, const TSInputEdit *edit, SubtreePool *); +char *ts_subtree_string(const Subtree *, const TSLanguage *, bool include_all); +void ts_subtree_print_dot_graph(const Subtree *, const TSLanguage *, FILE *); +Subtree *ts_subtree_last_external_token(Subtree *); +bool ts_subtree_external_token_state_eq(const Subtree *, const Subtree *); + +static inline uint32_t ts_subtree_total_bytes(const Subtree *self) { + return self->padding.bytes + self->size.bytes; +} + +static inline uint32_t ts_subtree_total_rows(const Subtree *self) { + return self->padding.extent.row + self->size.extent.row; +} + +static inline Length ts_subtree_total_size(const Subtree *self) { + return length_add(self->padding, self->size); +} + +#ifdef __cplusplus +} +#endif + +#endif // RUNTIME_TREE_H_ diff --git a/src/runtime/tree.h b/src/runtime/tree.h deleted file mode 100644 index 4fa95522..00000000 --- a/src/runtime/tree.h +++ /dev/null @@ -1,125 +0,0 @@ -#ifndef RUNTIME_TREE_H_ -#define RUNTIME_TREE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include "tree_sitter/parser.h" -#include "tree_sitter/runtime.h" -#include "runtime/length.h" -#include "runtime/array.h" -#include - -extern TSStateId TS_TREE_STATE_NONE; - -typedef struct { - union { - char *long_data; - char short_data[sizeof(char *) + sizeof(uint32_t)]; - }; - uint32_t length; -} TSExternalTokenState; - -typedef struct Tree Tree; - -typedef Array(Tree *) TreeArray; - -struct Tree { - Length padding; - Length size; - volatile uint32_t ref_count; - uint32_t bytes_scanned; - uint32_t error_cost; - uint32_t node_count; - uint32_t repeat_depth; - uint32_t child_count; - int32_t dynamic_precedence; - - bool visible : 1; - bool named : 1; - bool extra : 1; - bool fragile_left : 1; - bool fragile_right : 1; - bool has_changes : 1; - bool has_external_tokens : 1; - bool is_missing : 1; - TSSymbol symbol; - TSStateId parse_state; - struct { - TSSymbol symbol; - TSLexMode lex_mode; - } first_leaf; - - union { - struct { - TreeArray children; - uint32_t visible_child_count; - uint32_t named_child_count; - uint16_t alias_sequence_id; - }; - struct { - uint32_t _2; - TSExternalTokenState external_token_state; - }; - struct { - uint32_t _1; - int32_t lookahead_char; - }; - }; -}; - -typedef struct { - TreeArray free_trees; - TreeArray tree_stack; -} TreePool; - -void ts_external_token_state_init(TSExternalTokenState *, const char *, unsigned); -const char *ts_external_token_state_data(const TSExternalTokenState *); - -bool ts_tree_array_copy(TreeArray, TreeArray *); -void ts_tree_array_delete(TreePool *, TreeArray *); -TreeArray ts_tree_array_remove_trailing_extras(TreeArray *); -void ts_tree_array_reverse(TreeArray *); - -void ts_tree_pool_init(TreePool *); -void ts_tree_pool_delete(TreePool *); -Tree *ts_tree_pool_allocate(TreePool *); -void ts_tree_pool_free(TreePool *, Tree *); - -Tree *ts_tree_make_leaf(TreePool *, TSSymbol, Length, Length, const TSLanguage *); -Tree *ts_tree_make_node(TreePool *, TSSymbol, TreeArray *, unsigned, const TSLanguage *); -Tree *ts_tree_make_copy(TreePool *, Tree *child); -Tree *ts_tree_make_error_node(TreePool *, TreeArray *, const TSLanguage *); -Tree *ts_tree_make_error(TreePool *, Length, Length, int32_t, const TSLanguage *); -Tree *ts_tree_make_missing_leaf(TreePool *, TSSymbol, const TSLanguage *); -void ts_tree_retain(Tree *tree); -void ts_tree_release(TreePool *, Tree *tree); -bool ts_tree_eq(const Tree *tree1, const Tree *tree2); -int ts_tree_compare(const Tree *tree1, const Tree *tree2); -void ts_tree_set_children(Tree *, TreeArray *, const TSLanguage *); -void ts_tree_balance(Tree *, TreePool *, const TSLanguage *); -Tree *ts_tree_edit(Tree *, const TSInputEdit *edit, TreePool *); -char *ts_tree_string(const Tree *, const TSLanguage *, bool include_all); -void ts_tree_print_dot_graph(const Tree *, const TSLanguage *, FILE *); -Tree *ts_tree_last_external_token(Tree *); -bool ts_tree_external_token_state_eq(const Tree *, const Tree *); - -static inline uint32_t ts_tree_total_bytes(const Tree *self) { - return self->padding.bytes + self->size.bytes; -} - -static inline uint32_t ts_tree_total_rows(const Tree *self) { - return self->padding.extent.row + self->size.extent.row; -} - -static inline Length ts_tree_total_size(const Tree *self) { - return length_add(self->padding, self->size); -} - -#ifdef __cplusplus -} -#endif - -#endif // RUNTIME_TREE_H_ diff --git a/src/runtime/tree_cursor.c b/src/runtime/tree_cursor.c index db8298d2..730dacf9 100644 --- a/src/runtime/tree_cursor.c +++ b/src/runtime/tree_cursor.c @@ -24,7 +24,7 @@ void ts_tree_cursor_delete(TSTreeCursor *self) { bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { TreeCursorEntry *last_entry = array_back(&self->stack); - Tree *tree = last_entry->tree; + Subtree *tree = last_entry->tree; Length position = last_entry->position; bool did_descend; @@ -33,7 +33,7 @@ bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { uint32_t structural_child_index = 0; for (uint32_t i = 0; i < tree->children.size; i++) { - Tree *child = tree->children.contents[i]; + Subtree *child = tree->children.contents[i]; if (child->visible || child->visible_child_count > 0) { array_push(&self->stack, ((TreeCursorEntry) { .tree = child, @@ -51,7 +51,7 @@ bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { } } if (!child->extra) structural_child_index++; - position = length_add(position, ts_tree_total_size(child)); + position = length_add(position, ts_subtree_total_size(child)); } } while (did_descend); @@ -64,15 +64,15 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) { TreeCursorEntry *parent_entry = &self->stack.contents[i]; - Tree *parent = parent_entry->tree; + Subtree *parent = parent_entry->tree; uint32_t child_index = child_entry->child_index; uint32_t structural_child_index = child_entry->structural_child_index; Length position = child_entry->position; - Tree *child = parent->children.contents[child_index]; + Subtree *child = parent->children.contents[child_index]; while (++child_index < parent->children.size) { if (!child->extra) structural_child_index++; - position = length_add(position, ts_tree_total_size(child)); + position = length_add(position, ts_subtree_total_size(child)); child = parent->children.contents[child_index]; if (child->visible || child->visible_child_count > 0) { diff --git a/src/runtime/tree_cursor.h b/src/runtime/tree_cursor.h index 33e1d802..5e571fba 100644 --- a/src/runtime/tree_cursor.h +++ b/src/runtime/tree_cursor.h @@ -1,10 +1,10 @@ #ifndef RUNTIME_TREE_CURSOR_H_ #define RUNTIME_TREE_CURSOR_H_ -#include "runtime/tree.h" +#include "runtime/subtree.h" typedef struct { - Tree *tree; + Subtree *tree; Length position; uint32_t child_index; uint32_t structural_child_index; diff --git a/test/helpers/tree_helpers.cc b/test/helpers/tree_helpers.cc index 9a86b364..7fd015a3 100644 --- a/test/helpers/tree_helpers.cc +++ b/test/helpers/tree_helpers.cc @@ -14,21 +14,21 @@ const char *symbol_names[24] = { "twenty-two", "twenty-three" }; -TreeArray *tree_array(std::vector trees) { - static TreeArray result; +SubtreeArray *tree_array(std::vector trees) { + static SubtreeArray result; result.capacity = trees.size(); result.size = trees.size(); - result.contents = (Tree **)calloc(trees.size(), sizeof(Tree *)); + result.contents = (Subtree **)calloc(trees.size(), sizeof(Subtree *)); for (size_t i = 0; i < trees.size(); i++) { result.contents[i] = trees[i]; } return &result; } -ostream &operator<<(std::ostream &stream, const Tree *tree) { +ostream &operator<<(std::ostream &stream, const Subtree *tree) { static TSLanguage DUMMY_LANGUAGE = {}; DUMMY_LANGUAGE.symbol_names = symbol_names; - char *string = ts_tree_string(tree, &DUMMY_LANGUAGE, false); + char *string = ts_subtree_string(tree, &DUMMY_LANGUAGE, false); stream << string; ts_free(string); return stream; @@ -49,7 +49,7 @@ bool operator==(const TSNode &left, const TSNode &right) { return ts_node_eq(left, right); } -bool operator==(const std::vector &vec, const TreeArray &array) { +bool operator==(const std::vector &vec, const SubtreeArray &array) { if (vec.size() != array.size) return false; for (size_t i = 0; i < array.size; i++) diff --git a/test/helpers/tree_helpers.h b/test/helpers/tree_helpers.h index c28dcd98..bd08aad7 100644 --- a/test/helpers/tree_helpers.h +++ b/test/helpers/tree_helpers.h @@ -1,17 +1,17 @@ #ifndef HELPERS_TREE_HELPERS_H_ #define HELPERS_TREE_HELPERS_H_ -#include "runtime/tree.h" +#include "runtime/subtree.h" #include #include extern const char *symbol_names[24]; -TreeArray *tree_array(std::vector trees); +SubtreeArray *tree_array(std::vector trees); -std::ostream &operator<<(std::ostream &stream, const Tree *tree); +std::ostream &operator<<(std::ostream &stream, const Subtree *tree); std::ostream &operator<<(std::ostream &stream, const TSNode &node); bool operator==(const TSNode &left, const TSNode &right); -bool operator==(const std::vector &right, const TreeArray &array); +bool operator==(const std::vector &right, const SubtreeArray &array); void assert_consistent_tree_sizes(TSNode node); diff --git a/test/runtime/stack_test.cc b/test/runtime/stack_test.cc index d6cbcea0..99a979d1 100644 --- a/test/runtime/stack_test.cc +++ b/test/runtime/stack_test.cc @@ -4,7 +4,7 @@ #include "helpers/record_alloc.h" #include "helpers/stream_methods.h" #include "runtime/stack.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/length.h" #include "runtime/alloc.h" @@ -23,23 +23,23 @@ Length operator*(const Length &length, uint32_t factor) { return {length.bytes * factor, {0, length.extent.column * factor}}; } -void free_slice_array(TreePool *pool, StackSliceArray *slices) { +void free_slice_array(SubtreePool *pool, StackSliceArray *slices) { for (size_t i = 0; i < slices->size; i++) { StackSlice slice = slices->contents[i]; bool matches_prior_trees = false; for (size_t j = 0; j < i; j++) { StackSlice prior_slice = slices->contents[j]; - if (slice.trees.contents == prior_slice.trees.contents) { + if (slice.subtrees.contents == prior_slice.subtrees.contents) { matches_prior_trees = true; break; } } if (!matches_prior_trees) { - for (size_t j = 0; j < slice.trees.size; j++) - ts_tree_release(pool, slice.trees.contents[j]); - array_delete(&slice.trees); + for (size_t j = 0; j < slice.subtrees.size; j++) + ts_subtree_release(pool, slice.subtrees.contents[j]); + array_delete(&slice.subtrees); } } } @@ -54,9 +54,9 @@ vector get_stack_entries(Stack *stack, StackVersion version) { ts_stack_iterate( stack, version, - [](void *payload, TSStateId state, uint32_t tree_count) { + [](void *payload, TSStateId state, uint32_t subtree_count) { auto entries = static_cast *>(payload); - StackEntry entry = {state, tree_count}; + StackEntry entry = {state, subtree_count}; if (find(entries->begin(), entries->end(), entry) == entries->end()) { entries->push_back(entry); } @@ -68,39 +68,39 @@ START_TEST describe("Stack", [&]() { Stack *stack; - const size_t tree_count = 11; - Tree *trees[tree_count]; + const size_t subtree_count = 11; + Subtree *subtrees[subtree_count]; Length tree_len = {3, {0, 3}}; - TreePool pool; + SubtreePool pool; before_each([&]() { record_alloc::start(); - ts_tree_pool_init(&pool); + ts_subtree_pool_init(&pool); stack = ts_stack_new(&pool); TSLanguage dummy_language; TSSymbolMetadata symbol_metadata[50] = {}; dummy_language.symbol_metadata = symbol_metadata; - for (size_t i = 0; i < tree_count; i++) { - trees[i] = ts_tree_make_leaf(&pool, i, length_zero(), tree_len, &dummy_language); + for (size_t i = 0; i < subtree_count; i++) { + subtrees[i] = ts_subtree_make_leaf(&pool, i, length_zero(), tree_len, &dummy_language); } }); after_each([&]() { ts_stack_delete(stack); - for (size_t i = 0; i < tree_count; i++) { - ts_tree_release(&pool, trees[i]); + for (size_t i = 0; i < subtree_count; i++) { + ts_subtree_release(&pool, subtrees[i]); } - ts_tree_pool_delete(&pool); + ts_subtree_pool_delete(&pool); record_alloc::stop(); AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); }); - auto push = [&](StackVersion version, Tree *tree, TSStateId state) { - ts_tree_retain(tree); + auto push = [&](StackVersion version, Subtree *tree, TSStateId state) { + ts_subtree_retain(tree); ts_stack_push(stack, version, tree, false, state); }; @@ -111,17 +111,17 @@ describe("Stack", [&]() { AssertThat(ts_stack_position(stack, 0), Equals(length_zero())); // . <──0── A* - push(0, trees[0], stateA); + push(0, subtrees[0], stateA); AssertThat(ts_stack_state(stack, 0), Equals(stateA)); AssertThat(ts_stack_position(stack, 0), Equals(tree_len)); // . <──0── A <──1── B* - push(0, trees[1], stateB); + push(0, subtrees[1], stateB); AssertThat(ts_stack_state(stack, 0), Equals(stateB)); AssertThat(ts_stack_position(stack, 0), Equals(tree_len * 2)); // . <──0── A <──1── B <──2── C* - push(0, trees[2], stateC); + push(0, subtrees[2], stateC); AssertThat(ts_stack_state(stack, 0), Equals(stateC)); AssertThat(ts_stack_position(stack, 0), Equals(tree_len * 3)); @@ -139,7 +139,7 @@ describe("Stack", [&]() { // . <──0── A <─* // ↑ // └───* - push(0, trees[0], stateA); + push(0, subtrees[0], stateA); ts_stack_copy_version(stack, 0); }); @@ -147,10 +147,10 @@ describe("Stack", [&]() { // . <──0── A <──1── B <──3── D* // ↑ // └───2─── C <──4── D* - push(0, trees[1], stateB); - push(1, trees[2], stateC); - push(0, trees[3], stateD); - push(1, trees[4], stateD); + push(0, subtrees[1], stateB); + push(1, subtrees[2], stateC); + push(0, subtrees[3], stateD); + push(1, subtrees[4], stateD); // . <──0── A <──1── B <──3── D* // ↑ | @@ -170,8 +170,8 @@ describe("Stack", [&]() { // . <──0── A <──1── B* // ↑ // └───2─── C* - push(0, trees[1], stateB); - push(1, trees[2], stateC); + push(0, subtrees[1], stateB); + push(1, subtrees[2], stateC); AssertThat(ts_stack_merge(stack, 0, 1), IsFalse()); AssertThat(ts_stack_version_count(stack), Equals(2)); @@ -181,11 +181,11 @@ describe("Stack", [&]() { // . <──0── A <──1── B <────3──── D* // ↑ // └───2─── C <──4── D* - trees[3]->size = tree_len * 3; - push(0, trees[1], stateB); - push(1, trees[2], stateC); - push(0, trees[3], stateD); - push(1, trees[4], stateD); + subtrees[3]->size = tree_len * 3; + push(0, subtrees[1], stateB); + push(1, subtrees[2], stateC); + push(0, subtrees[3], stateD); + push(1, subtrees[4], stateD); AssertThat(ts_stack_merge(stack, 0, 1), IsFalse()); AssertThat(ts_stack_version_count(stack), Equals(2)); @@ -196,12 +196,12 @@ describe("Stack", [&]() { // . <──0── A <──1── B <──3── D <──5── E* // ↑ // └───2─── C <──4── D <──5── E* - push(0, trees[1], stateB); - push(1, trees[2], stateC); - push(0, trees[3], stateD); - push(1, trees[4], stateD); - push(0, trees[5], stateE); - push(1, trees[5], stateE); + push(0, subtrees[1], stateB); + push(1, subtrees[2], stateC); + push(0, subtrees[3], stateD); + push(1, subtrees[4], stateD); + push(0, subtrees[5], stateE); + push(1, subtrees[5], stateE); // . <──0── A <──1── B <──3── D <──5── E* // ↑ | @@ -224,12 +224,12 @@ describe("Stack", [&]() { // . <──0── A <────1──── B* // ↑ // └2─ A <──1── B* - trees[2]->extra = true; - trees[2]->size = tree_len * 0; + subtrees[2]->extra = true; + subtrees[2]->size = tree_len * 0; - push(0, trees[1], stateB); - push(1, trees[2], stateA); - push(1, trees[1], stateB); + push(0, subtrees[1], stateB); + push(1, subtrees[2], stateA); + push(1, subtrees[1], stateB); // . <──0── A <──1── B* AssertThat(ts_stack_merge(stack, 0, 1), IsTrue()); @@ -246,9 +246,9 @@ describe("Stack", [&]() { describe("pop_count(version, count)", [&]() { before_each([&]() { // . <──0── A <──1── B <──2── C* - push(0, trees[0], stateA); - push(0, trees[1], stateB); - push(0, trees[2], stateC); + push(0, subtrees[0], stateA); + push(0, subtrees[1], stateB); + push(0, subtrees[2], stateC); }); it("creates a new version with the given number of entries removed", [&]() { @@ -261,14 +261,14 @@ describe("Stack", [&]() { StackSlice slice = pop.contents[0]; AssertThat(slice.version, Equals(1)); - AssertThat(slice.trees, Equals(vector({ trees[1], trees[2] }))); + AssertThat(slice.subtrees, Equals(vector({ subtrees[1], subtrees[2] }))); AssertThat(ts_stack_state(stack, 1), Equals(stateA)); free_slice_array(&pool,&pop); }); - it("does not count 'extra' trees toward the given count", [&]() { - trees[1]->extra = true; + it("does not count 'extra' subtrees toward the given count", [&]() { + subtrees[1]->extra = true; // . <──0── A <──1── B <──2── C* // ↑ @@ -277,7 +277,7 @@ describe("Stack", [&]() { AssertThat(pop.size, Equals(1)); StackSlice slice = pop.contents[0]; - AssertThat(slice.trees, Equals(vector({ trees[0], trees[1], trees[2] }))); + AssertThat(slice.subtrees, Equals(vector({ subtrees[0], subtrees[1], subtrees[2] }))); AssertThat(ts_stack_state(stack, 1), Equals(1)); free_slice_array(&pool,&pop); @@ -288,14 +288,14 @@ describe("Stack", [&]() { // . <──0── A <──1── B <──2── C <──3── D <──10── I* // ↑ | // └───4─── E <──5── F <──6───┘ - push(0, trees[3], stateD); + push(0, subtrees[3], stateD); StackSliceArray pop = ts_stack_pop_count(stack, 0, 3); free_slice_array(&pool,&pop); - push(1, trees[4], stateE); - push(1, trees[5], stateF); - push(1, trees[6], stateD); + push(1, subtrees[4], stateE); + push(1, subtrees[5], stateF); + push(1, subtrees[6], stateD); ts_stack_merge(stack, 0, 1); - push(0, trees[10], stateI); + push(0, subtrees[10], stateI); AssertThat(ts_stack_version_count(stack), Equals(1)); AssertThat(get_stack_entries(stack, 0), Equals(vector({ @@ -322,11 +322,11 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.trees, Equals(vector({ trees[2], trees[3], trees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[2], subtrees[3], subtrees[10] }))); StackSlice slice2 = pop.contents[1]; AssertThat(slice2.version, Equals(2)); - AssertThat(slice2.trees, Equals(vector({ trees[5], trees[6], trees[10] }))); + AssertThat(slice2.subtrees, Equals(vector({ subtrees[5], subtrees[6], subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(3)); AssertThat(get_stack_entries(stack, 0), Equals(vector({ @@ -366,7 +366,7 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.trees, Equals(vector({ trees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(2)); AssertThat(ts_stack_state(stack, 0), Equals(stateI)); @@ -388,11 +388,11 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.trees, Equals(vector({ trees[1], trees[2], trees[3], trees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[1], subtrees[2], subtrees[3], subtrees[10] }))); StackSlice slice2 = pop.contents[1]; AssertThat(slice2.version, Equals(1)); - AssertThat(slice2.trees, Equals(vector({ trees[4], trees[5], trees[6], trees[10] }))); + AssertThat(slice2.subtrees, Equals(vector({ subtrees[4], subtrees[5], subtrees[6], subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(2)); AssertThat(ts_stack_state(stack, 0), Equals(stateI)); @@ -403,7 +403,7 @@ describe("Stack", [&]() { }); describe("when there are three paths that lead to three different versions", [&]() { - it("returns three entries with different arrays of trees", [&]() { + it("returns three entries with different arrays of subtrees", [&]() { // . <──0── A <──1── B <──2── C <──3── D <──10── I* // ↑ | // ├───4─── E <──5── F <──6───┘ @@ -411,10 +411,10 @@ describe("Stack", [&]() { // └───7─── G <──8── H <──9───┘ StackSliceArray pop = ts_stack_pop_count(stack, 0, 4); free_slice_array(&pool,&pop); - push(1, trees[7], stateG); - push(1, trees[8], stateH); - push(1, trees[9], stateD); - push(1, trees[10], stateI); + push(1, subtrees[7], stateG); + push(1, subtrees[8], stateH); + push(1, subtrees[9], stateD); + push(1, subtrees[10], stateI); ts_stack_merge(stack, 0, 1); AssertThat(ts_stack_version_count(stack), Equals(1)); @@ -443,15 +443,15 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.trees, Equals(vector({ trees[3], trees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[3], subtrees[10] }))); StackSlice slice2 = pop.contents[1]; AssertThat(slice2.version, Equals(2)); - AssertThat(slice2.trees, Equals(vector({ trees[6], trees[10] }))); + AssertThat(slice2.subtrees, Equals(vector({ subtrees[6], subtrees[10] }))); StackSlice slice3 = pop.contents[2]; AssertThat(slice3.version, Equals(3)); - AssertThat(slice3.trees, Equals(vector({ trees[9], trees[10] }))); + AssertThat(slice3.subtrees, Equals(vector({ subtrees[9], subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(4)); AssertThat(ts_stack_state(stack, 0), Equals(stateI)); @@ -467,12 +467,12 @@ describe("Stack", [&]() { describe("pop_pending(version)", [&]() { before_each([&]() { - push(0, trees[0], stateA); + push(0, subtrees[0], stateA); }); it("removes the top node from the stack if it was pushed in pending mode", [&]() { - ts_stack_push(stack, 0, trees[1], true, stateB); - ts_tree_retain(trees[1]); + ts_stack_push(stack, 0, subtrees[1], true, stateB); + ts_subtree_retain(subtrees[1]); StackSliceArray pop = ts_stack_pop_pending(stack, 0); AssertThat(pop.size, Equals(1)); @@ -485,20 +485,20 @@ describe("Stack", [&]() { free_slice_array(&pool,&pop); }); - it("skips entries whose trees are extra", [&]() { - ts_stack_push(stack, 0, trees[1], true, stateB); - ts_tree_retain(trees[1]); + it("skips entries whose subtrees are extra", [&]() { + ts_stack_push(stack, 0, subtrees[1], true, stateB); + ts_subtree_retain(subtrees[1]); - trees[2]->extra = true; - trees[3]->extra = true; + subtrees[2]->extra = true; + subtrees[3]->extra = true; - push(0, trees[2], stateB); - push(0, trees[3], stateB); + push(0, subtrees[2], stateB); + push(0, subtrees[3], stateB); StackSliceArray pop = ts_stack_pop_pending(stack, 0); AssertThat(pop.size, Equals(1)); - AssertThat(pop.contents[0].trees, Equals(vector({ trees[1], trees[2], trees[3] }))); + AssertThat(pop.contents[0].subtrees, Equals(vector({ subtrees[1], subtrees[2], subtrees[3] }))); AssertThat(get_stack_entries(stack, 0), Equals(vector({ {stateA, 0}, @@ -509,7 +509,7 @@ describe("Stack", [&]() { }); it("does nothing if the top node was not pushed in pending mode", [&]() { - push(0, trees[1], stateB); + push(0, subtrees[1], stateB); StackSliceArray pop = ts_stack_pop_pending(stack, 0); AssertThat(pop.size, Equals(0)); @@ -526,59 +526,59 @@ describe("Stack", [&]() { describe("setting external token state", [&]() { before_each([&]() { - trees[1]->has_external_tokens = true; - trees[2]->has_external_tokens = true; - ts_external_token_state_init(&trees[1]->external_token_state, NULL, 0); - ts_external_token_state_init(&trees[2]->external_token_state, NULL, 0); + subtrees[1]->has_external_tokens = true; + subtrees[2]->has_external_tokens = true; + ts_external_token_state_init(&subtrees[1]->external_token_state, NULL, 0); + ts_external_token_state_init(&subtrees[2]->external_token_state, NULL, 0); }); it("allows the state to be retrieved", [&]() { - AssertThat(ts_stack_last_external_token(stack, 0), Equals(nullptr)); + AssertThat(ts_stack_last_external_token(stack, 0), Equals(nullptr)); - ts_stack_set_last_external_token(stack, 0, trees[1]); - AssertThat(ts_stack_last_external_token(stack, 0), Equals(trees[1])); + ts_stack_set_last_external_token(stack, 0, subtrees[1]); + AssertThat(ts_stack_last_external_token(stack, 0), Equals(subtrees[1])); ts_stack_copy_version(stack, 0); - AssertThat(ts_stack_last_external_token(stack, 1), Equals(trees[1])); + AssertThat(ts_stack_last_external_token(stack, 1), Equals(subtrees[1])); - ts_stack_set_last_external_token(stack, 0, trees[2]); - AssertThat(ts_stack_last_external_token(stack, 0), Equals(trees[2])); + ts_stack_set_last_external_token(stack, 0, subtrees[2]); + AssertThat(ts_stack_last_external_token(stack, 0), Equals(subtrees[2])); }); it("does not merge stack versions with different external token states", [&]() { - ts_external_token_state_init(&trees[1]->external_token_state, "abcd", 2); - ts_external_token_state_init(&trees[2]->external_token_state, "ABCD", 2); + ts_external_token_state_init(&subtrees[1]->external_token_state, "abcd", 2); + ts_external_token_state_init(&subtrees[2]->external_token_state, "ABCD", 2); ts_stack_copy_version(stack, 0); - push(0, trees[0], 5); - push(1, trees[0], 5); + push(0, subtrees[0], 5); + push(1, subtrees[0], 5); - ts_stack_set_last_external_token(stack, 0, trees[1]); - ts_stack_set_last_external_token(stack, 1, trees[2]); + ts_stack_set_last_external_token(stack, 0, subtrees[1]); + ts_stack_set_last_external_token(stack, 1, subtrees[2]); AssertThat(ts_stack_merge(stack, 0, 1), IsFalse()); }); it("merges stack versions with identical external token states", [&]() { - ts_external_token_state_init(&trees[1]->external_token_state, "abcd", 2); - ts_external_token_state_init(&trees[2]->external_token_state, "abcd", 2); + ts_external_token_state_init(&subtrees[1]->external_token_state, "abcd", 2); + ts_external_token_state_init(&subtrees[2]->external_token_state, "abcd", 2); ts_stack_copy_version(stack, 0); - push(0, trees[0], 5); - push(1, trees[0], 5); + push(0, subtrees[0], 5); + push(1, subtrees[0], 5); - ts_stack_set_last_external_token(stack, 0, trees[1]); - ts_stack_set_last_external_token(stack, 1, trees[2]); + ts_stack_set_last_external_token(stack, 0, subtrees[1]); + ts_stack_set_last_external_token(stack, 1, subtrees[2]); AssertThat(ts_stack_merge(stack, 0, 1), IsTrue()); }); it("does not distinguish between an *empty* external token state and *no* external token state", [&]() { ts_stack_copy_version(stack, 0); - push(0, trees[0], 5); - push(1, trees[0], 5); + push(0, subtrees[0], 5); + push(1, subtrees[0], 5); - ts_stack_set_last_external_token(stack, 0, trees[1]); + ts_stack_set_last_external_token(stack, 0, subtrees[1]); AssertThat(ts_stack_merge(stack, 0, 1), IsTrue()); }); @@ -595,7 +595,7 @@ std::ostream &operator<<(std::ostream &stream, const StackEntry &entry) { return stream << "{" << entry.state << ", " << entry.depth << "}"; } -std::ostream &operator<<(std::ostream &stream, const TreeArray &array) { +std::ostream &operator<<(std::ostream &stream, const SubtreeArray &array) { stream << "["; bool first = true; for (size_t i = 0; i < array.size; i++) { diff --git a/test/runtime/tree_test.cc b/test/runtime/subtree_test.cc similarity index 65% rename from test/runtime/tree_test.cc rename to test/runtime/subtree_test.cc index 1613b930..d56be01d 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/subtree_test.cc @@ -1,26 +1,26 @@ #include "test_helper.h" #include "helpers/tree_helpers.h" #include "helpers/point_helpers.h" -#include "runtime/tree.h" +#include "runtime/subtree.h" #include "runtime/length.h" -void assert_consistent(const Tree *tree) { +void assert_consistent(const Subtree *tree) { if (tree->child_count == 0) return; AssertThat(tree->children.contents[0]->padding, Equals(tree->padding)); Length total_children_size = length_zero(); for (size_t i = 0; i < tree->children.size; i++) { - Tree *child = tree->children.contents[i]; + Subtree *child = tree->children.contents[i]; assert_consistent(child); - total_children_size = length_add(total_children_size, ts_tree_total_size(child)); + total_children_size = length_add(total_children_size, ts_subtree_total_size(child)); } - AssertThat(total_children_size, Equals(ts_tree_total_size(tree))); + AssertThat(total_children_size, Equals(ts_subtree_total_size(tree))); }; START_TEST -describe("Tree", []() { +describe("Subtree", []() { enum { symbol1 = 1, symbol2, @@ -38,29 +38,29 @@ describe("Tree", []() { TSLanguage language; language.symbol_metadata = metadata_list; - TreePool pool; + SubtreePool pool; before_each([&]() { - ts_tree_pool_init(&pool); + ts_subtree_pool_init(&pool); }); after_each([&]() { - ts_tree_pool_delete(&pool); + ts_subtree_pool_delete(&pool); }); describe("make_leaf", [&]() { it("does not mark the tree as fragile", [&]() { - Tree *tree = ts_tree_make_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); + Subtree *tree = ts_subtree_make_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); AssertThat(tree->fragile_left, IsFalse()); AssertThat(tree->fragile_right, IsFalse()); - ts_tree_release(&pool, tree); + ts_subtree_release(&pool, tree); }); }); describe("make_error", [&]() { it("marks the tree as fragile", [&]() { - Tree *error_tree = ts_tree_make_error( + Subtree *error_tree = ts_subtree_make_error( &pool, length_zero(), length_zero(), @@ -71,29 +71,29 @@ describe("Tree", []() { AssertThat(error_tree->fragile_left, IsTrue()); AssertThat(error_tree->fragile_right, IsTrue()); - ts_tree_release(&pool, error_tree); + ts_subtree_release(&pool, error_tree); }); }); describe("make_node", [&]() { - Tree *tree1, *tree2, *parent1; + Subtree *tree1, *tree2, *parent1; before_each([&]() { - tree1 = ts_tree_make_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); - tree2 = ts_tree_make_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); + tree1 = ts_subtree_make_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); + tree2 = ts_subtree_make_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); - ts_tree_retain(tree1); - ts_tree_retain(tree2); - parent1 = ts_tree_make_node(&pool, symbol3, tree_array({ + ts_subtree_retain(tree1); + ts_subtree_retain(tree2); + parent1 = ts_subtree_make_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); }); after_each([&]() { - ts_tree_release(&pool, tree1); - ts_tree_release(&pool, tree2); - ts_tree_release(&pool, parent1); + ts_subtree_release(&pool, tree1); + ts_subtree_release(&pool, tree2); + ts_subtree_release(&pool, parent1); }); it("computes its size and padding based on its child nodes", [&]() { @@ -104,22 +104,22 @@ describe("Tree", []() { }); describe("when the first node is fragile on the left side", [&]() { - Tree *parent; + Subtree *parent; before_each([&]() { tree1->fragile_left = true; tree1->extra = true; - ts_tree_retain(tree1); - ts_tree_retain(tree2); - parent = ts_tree_make_node(&pool, symbol3, tree_array({ + ts_subtree_retain(tree1); + ts_subtree_retain(tree2); + parent = ts_subtree_make_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); }); after_each([&]() { - ts_tree_release(&pool, parent); + ts_subtree_release(&pool, parent); }); it("records that it is fragile on the left side", [&]() { @@ -128,22 +128,22 @@ describe("Tree", []() { }); describe("when the last node is fragile on the right side", [&]() { - Tree *parent; + Subtree *parent; before_each([&]() { tree2->fragile_right = true; tree2->extra = true; - ts_tree_retain(tree1); - ts_tree_retain(tree2); - parent = ts_tree_make_node(&pool, symbol3, tree_array({ + ts_subtree_retain(tree1); + ts_subtree_retain(tree2); + parent = ts_subtree_make_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); }); after_each([&]() { - ts_tree_release(&pool, parent); + ts_subtree_release(&pool, parent); }); it("records that it is fragile on the right side", [&]() { @@ -152,22 +152,22 @@ describe("Tree", []() { }); describe("when the outer nodes aren't fragile on their outer side", [&]() { - Tree *parent; + Subtree *parent; before_each([&]() { tree1->fragile_right = true; tree2->fragile_left = true; - ts_tree_retain(tree1); - ts_tree_retain(tree2); - parent = ts_tree_make_node(&pool, symbol3, tree_array({ + ts_subtree_retain(tree1); + ts_subtree_retain(tree2); + parent = ts_subtree_make_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); }); after_each([&]() { - ts_tree_release(&pool, parent); + ts_subtree_release(&pool, parent); }); it("records that it is not fragile", [&]() { @@ -178,13 +178,13 @@ describe("Tree", []() { }); describe("edit", [&]() { - Tree *tree; + Subtree *tree; before_each([&]() { - tree = ts_tree_make_node(&pool, symbol1, tree_array({ - ts_tree_make_leaf(&pool, symbol2, {2, {0, 2}}, {3, {0, 3}}, &language), - ts_tree_make_leaf(&pool, symbol3, {2, {0, 2}}, {3, {0, 3}}, &language), - ts_tree_make_leaf(&pool, symbol4, {2, {0, 2}}, {3, {0, 3}}, &language), + tree = ts_subtree_make_node(&pool, symbol1, tree_array({ + ts_subtree_make_leaf(&pool, symbol2, {2, {0, 2}}, {3, {0, 3}}, &language), + ts_subtree_make_leaf(&pool, symbol3, {2, {0, 2}}, {3, {0, 3}}, &language), + ts_subtree_make_leaf(&pool, symbol4, {2, {0, 2}}, {3, {0, 3}}, &language), }), 0, &language); AssertThat(tree->padding, Equals({2, {0, 2}})); @@ -192,7 +192,7 @@ describe("Tree", []() { }); after_each([&]() { - ts_tree_release(&pool, tree); + ts_subtree_release(&pool, tree); }); it("does not mutate the argument", [&]() { @@ -204,8 +204,8 @@ describe("Tree", []() { edit.extent_removed = {0, 0}; edit.extent_added = {0, 1}; - ts_tree_retain(tree); - Tree *new_tree = ts_tree_edit(tree, &edit, &pool); + ts_subtree_retain(tree); + Subtree *new_tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); assert_consistent(new_tree); @@ -221,7 +221,7 @@ describe("Tree", []() { AssertThat(tree->children.contents[1]->padding, Equals({2, {0, 2}})); AssertThat(tree->children.contents[1]->size, Equals({3, {0, 3}})); - ts_tree_release(&pool, new_tree); + ts_subtree_release(&pool, new_tree); }); describe("edits within a tree's padding", [&]() { @@ -233,7 +233,7 @@ describe("Tree", []() { edit.start_point = {0, 1}; edit.extent_removed = {0, 0}; edit.extent_added = {0, 1}; - tree = ts_tree_edit(tree, &edit, &pool); + tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -259,7 +259,7 @@ describe("Tree", []() { edit.start_point = {0, 1}; edit.extent_removed = {0, 3}; edit.extent_added = {0, 4}; - tree = ts_tree_edit(tree, &edit, &pool); + tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -281,7 +281,7 @@ describe("Tree", []() { edit.start_point = {0, 2}; edit.extent_removed = {0, 0}; edit.extent_added = {0, 2}; - tree = ts_tree_edit(tree, &edit, &pool); + tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -305,7 +305,7 @@ describe("Tree", []() { edit.start_point = {0, 2}; edit.extent_removed = {0, 2}; edit.extent_added = {0, 5}; - tree = ts_tree_edit(tree, &edit, &pool); + tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -329,7 +329,7 @@ describe("Tree", []() { edit.start_point = {0, 1}; edit.extent_removed = {0, 10}; edit.extent_added = {0, 3}; - tree = ts_tree_edit(tree, &edit, &pool); + tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->has_changes, IsTrue()); @@ -361,7 +361,7 @@ describe("Tree", []() { edit.start_point = {0, 6}; edit.extent_removed = {0, 1}; edit.extent_added = {0, 1}; - tree = ts_tree_edit(tree, &edit, &pool); + tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); AssertThat(tree->children.contents[0]->has_changes, IsTrue()); @@ -370,43 +370,43 @@ describe("Tree", []() { }); describe("eq", [&]() { - Tree *leaf; + Subtree *leaf; before_each([&]() { - leaf = ts_tree_make_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); + leaf = ts_subtree_make_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); }); after_each([&]() { - ts_tree_release(&pool, leaf); + ts_subtree_release(&pool, leaf); }); it("returns true for identical trees", [&]() { - Tree *leaf_copy = ts_tree_make_leaf(&pool, symbol1, {2, {1, 1}}, {5, {1, 4}}, &language); - AssertThat(ts_tree_eq(leaf, leaf_copy), IsTrue()); + Subtree *leaf_copy = ts_subtree_make_leaf(&pool, symbol1, {2, {1, 1}}, {5, {1, 4}}, &language); + AssertThat(ts_subtree_eq(leaf, leaf_copy), IsTrue()); - Tree *parent = ts_tree_make_node(&pool, symbol2, tree_array({ + Subtree *parent = ts_subtree_make_node(&pool, symbol2, tree_array({ leaf, leaf_copy, }), 0, &language); - ts_tree_retain(leaf); - ts_tree_retain(leaf_copy); + ts_subtree_retain(leaf); + ts_subtree_retain(leaf_copy); - Tree *parent_copy = ts_tree_make_node(&pool, symbol2, tree_array({ + Subtree *parent_copy = ts_subtree_make_node(&pool, symbol2, tree_array({ leaf, leaf_copy, }), 0, &language); - ts_tree_retain(leaf); - ts_tree_retain(leaf_copy); + ts_subtree_retain(leaf); + ts_subtree_retain(leaf_copy); - AssertThat(ts_tree_eq(parent, parent_copy), IsTrue()); + AssertThat(ts_subtree_eq(parent, parent_copy), IsTrue()); - ts_tree_release(&pool, leaf_copy); - ts_tree_release(&pool, parent); - ts_tree_release(&pool, parent_copy); + ts_subtree_release(&pool, leaf_copy); + ts_subtree_release(&pool, parent); + ts_subtree_release(&pool, parent_copy); }); it("returns false for trees with different symbols", [&]() { - Tree *different_leaf = ts_tree_make_leaf( + Subtree *different_leaf = ts_subtree_make_leaf( &pool, leaf->symbol + 1, leaf->padding, @@ -414,50 +414,50 @@ describe("Tree", []() { &language ); - AssertThat(ts_tree_eq(leaf, different_leaf), IsFalse()); - ts_tree_release(&pool, different_leaf); + AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); + ts_subtree_release(&pool, different_leaf); }); it("returns false for trees with different options", [&]() { - Tree *different_leaf = ts_tree_make_leaf(&pool, leaf->symbol, leaf->padding, leaf->size, &language); + Subtree *different_leaf = ts_subtree_make_leaf(&pool, leaf->symbol, leaf->padding, leaf->size, &language); different_leaf->visible = !leaf->visible; - AssertThat(ts_tree_eq(leaf, different_leaf), IsFalse()); - ts_tree_release(&pool, different_leaf); + AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); + ts_subtree_release(&pool, different_leaf); }); it("returns false for trees with different paddings or sizes", [&]() { - Tree *different_leaf = ts_tree_make_leaf(&pool, leaf->symbol, {}, leaf->size, &language); - AssertThat(ts_tree_eq(leaf, different_leaf), IsFalse()); - ts_tree_release(&pool, different_leaf); + Subtree *different_leaf = ts_subtree_make_leaf(&pool, leaf->symbol, {}, leaf->size, &language); + AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); + ts_subtree_release(&pool, different_leaf); - different_leaf = ts_tree_make_leaf(&pool, symbol1, leaf->padding, {}, &language); - AssertThat(ts_tree_eq(leaf, different_leaf), IsFalse()); - ts_tree_release(&pool, different_leaf); + different_leaf = ts_subtree_make_leaf(&pool, symbol1, leaf->padding, {}, &language); + AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); + ts_subtree_release(&pool, different_leaf); }); it("returns false for trees with different children", [&]() { - Tree *leaf2 = ts_tree_make_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); + Subtree *leaf2 = ts_subtree_make_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); - Tree *parent = ts_tree_make_node(&pool, symbol2, tree_array({ + Subtree *parent = ts_subtree_make_node(&pool, symbol2, tree_array({ leaf, leaf2, }), 0, &language); - ts_tree_retain(leaf); - ts_tree_retain(leaf2); + ts_subtree_retain(leaf); + ts_subtree_retain(leaf2); - Tree *different_parent = ts_tree_make_node(&pool, symbol2, tree_array({ + Subtree *different_parent = ts_subtree_make_node(&pool, symbol2, tree_array({ leaf2, leaf, }), 0, &language); - ts_tree_retain(leaf2); - ts_tree_retain(leaf); + ts_subtree_retain(leaf2); + ts_subtree_retain(leaf); - AssertThat(ts_tree_eq(different_parent, parent), IsFalse()); - AssertThat(ts_tree_eq(parent, different_parent), IsFalse()); + AssertThat(ts_subtree_eq(different_parent, parent), IsFalse()); + AssertThat(ts_subtree_eq(parent, different_parent), IsFalse()); - ts_tree_release(&pool, leaf2); - ts_tree_release(&pool, parent); - ts_tree_release(&pool, different_parent); + ts_subtree_release(&pool, leaf2); + ts_subtree_release(&pool, parent); + ts_subtree_release(&pool, different_parent); }); }); @@ -465,32 +465,32 @@ describe("Tree", []() { Length padding = {1, {0, 1}}; Length size = {2, {0, 2}}; - auto make_external = [](Tree *tree) { + auto make_external = [](Subtree *tree) { tree->has_external_tokens = true; return tree; }; it("returns the last serialized external token state in the given tree", [&]() { - Tree *tree1, *tree2, *tree3, *tree4, *tree5, *tree6, *tree7, *tree8, *tree9; + Subtree *tree1, *tree2, *tree3, *tree4, *tree5, *tree6, *tree7, *tree8, *tree9; - tree1 = ts_tree_make_node(&pool, symbol1, tree_array({ - (tree2 = ts_tree_make_node(&pool, symbol2, tree_array({ - (tree3 = make_external(ts_tree_make_leaf(&pool, symbol3, padding, size, &language))), - (tree4 = ts_tree_make_leaf(&pool, symbol4, padding, size, &language)), - (tree5 = ts_tree_make_leaf(&pool, symbol5, padding, size, &language)), + tree1 = ts_subtree_make_node(&pool, symbol1, tree_array({ + (tree2 = ts_subtree_make_node(&pool, symbol2, tree_array({ + (tree3 = make_external(ts_subtree_make_leaf(&pool, symbol3, padding, size, &language))), + (tree4 = ts_subtree_make_leaf(&pool, symbol4, padding, size, &language)), + (tree5 = ts_subtree_make_leaf(&pool, symbol5, padding, size, &language)), }), 0, &language)), - (tree6 = ts_tree_make_node(&pool, symbol6, tree_array({ - (tree7 = ts_tree_make_node(&pool, symbol7, tree_array({ - (tree8 = ts_tree_make_leaf(&pool, symbol8, padding, size, &language)), + (tree6 = ts_subtree_make_node(&pool, symbol6, tree_array({ + (tree7 = ts_subtree_make_node(&pool, symbol7, tree_array({ + (tree8 = ts_subtree_make_leaf(&pool, symbol8, padding, size, &language)), }), 0, &language)), - (tree9 = ts_tree_make_leaf(&pool, symbol9, padding, size, &language)), + (tree9 = ts_subtree_make_leaf(&pool, symbol9, padding, size, &language)), }), 0, &language)), }), 0, &language); - auto token = ts_tree_last_external_token(tree1); + auto token = ts_subtree_last_external_token(tree1); AssertThat(token, Equals(tree3)); - ts_tree_release(&pool, tree1); + ts_subtree_release(&pool, tree1); }); }); }); diff --git a/tests.gyp b/tests.gyp index eefdf2bf..b964a507 100644 --- a/tests.gyp +++ b/tests.gyp @@ -71,7 +71,7 @@ 'test/runtime/node_test.cc', 'test/runtime/parser_test.cc', 'test/runtime/stack_test.cc', - 'test/runtime/tree_test.cc', + 'test/runtime/subtree_test.cc', 'test/tests.cc', ], 'cflags': [ From 33f76430402f59cccafdd5052b92344e9a070564 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 10 May 2018 15:16:24 -0700 Subject: [PATCH 19/31] Rename Parser -> TSParser Co-Authored-By: Rick Winfrey --- src/runtime/document.c | 8 +-- src/runtime/document.h | 2 +- src/runtime/parser.c | 148 +++++++++++++++++++++-------------------- src/runtime/parser.h | 10 +-- 4 files changed, 85 insertions(+), 83 deletions(-) diff --git a/src/runtime/document.c b/src/runtime/document.c index 4298025a..7d175b6a 100644 --- a/src/runtime/document.c +++ b/src/runtime/document.c @@ -12,7 +12,7 @@ TSDocument *ts_document_new() { TSDocument *self = ts_calloc(1, sizeof(TSDocument)); - parser_init(&self->parser); + ts_parser_init(&self->parser); array_init(&self->cursor1.stack); array_init(&self->cursor2.stack); return self; @@ -22,7 +22,7 @@ void ts_document_free(TSDocument *self) { if (self->tree) ts_subtree_release(&self->parser.tree_pool, self->tree); if (self->cursor1.stack.contents) array_delete(&self->cursor1.stack); if (self->cursor2.stack.contents) array_delete(&self->cursor2.stack); - parser_destroy(&self->parser); + ts_parser_destroy(&self->parser); ts_document_set_input(self, (TSInput){ NULL, NULL, @@ -39,7 +39,7 @@ const TSLanguage *ts_document_language(TSDocument *self) { void ts_document_set_language(TSDocument *self, const TSLanguage *language) { if (language->version != TREE_SITTER_LANGUAGE_VERSION) return; ts_document_invalidate(self); - parser_set_language(&self->parser, language); + ts_parser_set_language(&self->parser, language); if (self->tree) { ts_subtree_release(&self->parser.tree_pool, self->tree); self->tree = NULL; @@ -134,7 +134,7 @@ void ts_document_parse_with_options(TSDocument *self, TSParseOptions options) { if (reusable_tree && !reusable_tree->has_changes) return; - Subtree *tree = parser_parse(&self->parser, self->input, reusable_tree, options.halt_on_error); + Subtree *tree = ts_parser_parse(&self->parser, self->input, reusable_tree, options.halt_on_error); if (self->tree) { Subtree *old_tree = self->tree; diff --git a/src/runtime/document.h b/src/runtime/document.h index 9e53963b..a7476ebf 100644 --- a/src/runtime/document.h +++ b/src/runtime/document.h @@ -7,7 +7,7 @@ #include struct TSDocument { - Parser parser; + TSParser parser; TSInput input; Subtree *tree; TSTreeCursor cursor1; diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 8bb7ae31..f3f6fd64 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -16,7 +16,7 @@ #define LOG(...) \ if (self->lexer.logger.log || self->print_debugging_graphs) { \ snprintf(self->lexer.debug_buffer, TREE_SITTER_SERIALIZATION_BUFFER_SIZE, __VA_ARGS__); \ - parser__log(self); \ + ts_parser__log(self); \ } #define LOG_STACK() \ @@ -25,10 +25,10 @@ fputs("\n\n", stderr); \ } -#define LOG_TREE() \ - if (self->print_debugging_graphs) { \ +#define LOG_TREE() \ + if (self->print_debugging_graphs) { \ ts_subtree_print_dot_graph(self->finished_tree, self->language, stderr); \ - fputs("\n", stderr); \ + fputs("\n", stderr); \ } #define SYM_NAME(symbol) ts_language_symbol_name(self->language, symbol) @@ -52,7 +52,7 @@ typedef enum { ErrorComparisonTakeRight, } ErrorComparison; -static void parser__log(Parser *self) { +static void ts_parser__log(TSParser *self) { if (self->lexer.logger.log) { self->lexer.logger.log( self->lexer.logger.payload, @@ -71,7 +71,7 @@ static void parser__log(Parser *self) { } } -static bool parser__breakdown_top_of_stack(Parser *self, StackVersion version) { +static bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion version) { bool did_break_down = false; bool pending = false; @@ -116,7 +116,7 @@ static bool parser__breakdown_top_of_stack(Parser *self, StackVersion version) { return did_break_down; } -static void parser__breakdown_lookahead(Parser *self, Subtree **lookahead, +static void ts_parser__breakdown_lookahead(TSParser *self, Subtree **lookahead, TSStateId state, ReusableNode *reusable_node) { bool did_descend = false; @@ -135,7 +135,7 @@ static void parser__breakdown_lookahead(Parser *self, Subtree **lookahead, } } -static ErrorComparison parser__compare_versions(Parser *self, ErrorStatus a, ErrorStatus b) { +static ErrorComparison ts_parser__compare_versions(TSParser *self, ErrorStatus a, ErrorStatus b) { if (!a.is_in_error && b.is_in_error) { if (a.cost < b.cost) { return ErrorComparisonTakeLeft; @@ -173,7 +173,7 @@ static ErrorComparison parser__compare_versions(Parser *self, ErrorStatus a, Err return ErrorComparisonNone; } -static ErrorStatus parser__version_status(Parser *self, StackVersion version) { +static ErrorStatus ts_parser__version_status(TSParser *self, StackVersion version) { unsigned cost = ts_stack_error_cost(self->stack, version); bool is_paused = ts_stack_is_paused(self->stack, version); if (is_paused) cost += ERROR_COST_PER_SKIPPED_TREE; @@ -185,7 +185,7 @@ static ErrorStatus parser__version_status(Parser *self, StackVersion version) { }; } -static bool parser__better_version_exists(Parser *self, StackVersion version, +static bool ts_parser__better_version_exists(TSParser *self, StackVersion version, bool is_in_error, unsigned cost) { if (self->finished_tree && self->finished_tree->error_cost <= cost) return true; @@ -201,8 +201,8 @@ static bool parser__better_version_exists(Parser *self, StackVersion version, if (i == version || !ts_stack_is_active(self->stack, i) || ts_stack_position(self->stack, i).bytes < position.bytes) continue; - ErrorStatus status_i = parser__version_status(self, i); - switch (parser__compare_versions(self, status, status_i)) { + ErrorStatus status_i = ts_parser__version_status(self, i); + switch (ts_parser__compare_versions(self, status, status_i)) { case ErrorComparisonTakeRight: return true; case ErrorComparisonPreferRight: @@ -215,7 +215,7 @@ static bool parser__better_version_exists(Parser *self, StackVersion version, return false; } -static void parser__restore_external_scanner(Parser *self, Subtree *external_token) { +static void ts_parser__restore_external_scanner(TSParser *self, Subtree *external_token) { if (external_token) { self->language->external_scanner.deserialize( self->external_scanner_payload, @@ -227,7 +227,7 @@ static void parser__restore_external_scanner(Parser *self, Subtree *external_tok } } -static Subtree *parser__lex(Parser *self, StackVersion version, TSStateId parse_state) { +static Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId parse_state) { Length start_position = ts_stack_position(self->stack, version); Subtree *external_token = ts_stack_last_external_token(self->stack, version); TSLexMode lex_mode = self->language->lex_modes[parse_state]; @@ -256,7 +256,7 @@ static Subtree *parser__lex(Parser *self, StackVersion version, TSStateId parse_ current_position.extent.column ); ts_lexer_start(&self->lexer); - parser__restore_external_scanner(self, external_token); + ts_parser__restore_external_scanner(self, external_token); if (self->language->external_scanner.scan( self->external_scanner_payload, &self->lexer.data, @@ -375,7 +375,8 @@ static Subtree *parser__lex(Parser *self, StackVersion version, TSStateId parse_ return result; } -static Subtree *parser__get_cached_token(Parser *self, size_t byte_index, Subtree *last_external_token) { +static Subtree *ts_parser__get_cached_token(TSParser *self, size_t byte_index, + Subtree *last_external_token) { TokenCache *cache = &self->token_cache; if (cache->token && cache->byte_index == byte_index && @@ -386,7 +387,7 @@ static Subtree *parser__get_cached_token(Parser *self, size_t byte_index, Subtre } } -static void parser__set_cached_token(Parser *self, size_t byte_index, Subtree *last_external_token, +static void ts_parser__set_cached_token(TSParser *self, size_t byte_index, Subtree *last_external_token, Subtree *token) { TokenCache *cache = &self->token_cache; if (token) ts_subtree_retain(token); @@ -398,7 +399,7 @@ static void parser__set_cached_token(Parser *self, size_t byte_index, Subtree *l cache->last_external_token = last_external_token; } -static bool parser__can_reuse_first_leaf(Parser *self, TSStateId state, Subtree *tree, +static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, Subtree *tree, TableEntry *table_entry) { TSLexMode current_lex_mode = self->language->lex_modes[state]; @@ -416,7 +417,7 @@ static bool parser__can_reuse_first_leaf(Parser *self, TSStateId state, Subtree return current_lex_mode.external_lex_state == 0 && table_entry->is_reusable; } -static Subtree *parser__get_lookahead(Parser *self, StackVersion version, TSStateId *state, +static Subtree *ts_parser__get_lookahead(TSParser *self, StackVersion version, TSStateId *state, ReusableNode *reusable_node, TableEntry *table_entry) { Length position = ts_stack_position(self->stack, version); Subtree *last_external_token = ts_stack_last_external_token(self->stack, version); @@ -458,14 +459,14 @@ static Subtree *parser__get_lookahead(Parser *self, StackVersion version, TSStat LOG("cant_reuse_node_%s tree:%s", reason, SYM_NAME(result->symbol)); if (!reusable_node_descend(reusable_node)) { reusable_node_advance(reusable_node); - parser__breakdown_top_of_stack(self, version); + ts_parser__breakdown_top_of_stack(self, version); *state = ts_stack_state(self->stack, version); } continue; } ts_language_table_entry(self->language, *state, result->first_leaf.symbol, table_entry); - if (!parser__can_reuse_first_leaf(self, *state, result, table_entry)) { + if (!ts_parser__can_reuse_first_leaf(self, *state, result, table_entry)) { LOG( "cant_reuse_node symbol:%s, first_leaf_symbol:%s", SYM_NAME(result->symbol), @@ -480,21 +481,21 @@ static Subtree *parser__get_lookahead(Parser *self, StackVersion version, TSStat return result; } - if ((result = parser__get_cached_token(self, position.bytes, last_external_token))) { + if ((result = ts_parser__get_cached_token(self, position.bytes, last_external_token))) { ts_language_table_entry(self->language, *state, result->first_leaf.symbol, table_entry); - if (parser__can_reuse_first_leaf(self, *state, result, table_entry)) { + if (ts_parser__can_reuse_first_leaf(self, *state, result, table_entry)) { ts_subtree_retain(result); return result; } } - result = parser__lex(self, version, *state); - parser__set_cached_token(self, position.bytes, last_external_token, result); + result = ts_parser__lex(self, version, *state); + ts_parser__set_cached_token(self, position.bytes, last_external_token, result); ts_language_table_entry(self->language, *state, result->symbol, table_entry); return result; } -static bool parser__select_tree(Parser *self, Subtree *left, Subtree *right) { +static bool ts_parser__select_tree(TSParser *self, Subtree *left, Subtree *right) { if (!left) return true; if (!right) return false; @@ -544,7 +545,7 @@ static bool parser__select_tree(Parser *self, Subtree *left, Subtree *right) { } } -static void parser__shift(Parser *self, StackVersion version, TSStateId state, +static void ts_parser__shift(TSParser *self, StackVersion version, TSStateId state, Subtree *lookahead, bool extra) { if (extra != lookahead->extra) { if (ts_stack_version_count(self->stack) > 1) { @@ -566,11 +567,11 @@ static void parser__shift(Parser *self, StackVersion version, TSStateId state, } } -static bool parser__replace_children(Parser *self, Subtree *tree, SubtreeArray *children) { +static bool ts_parser__replace_children(TSParser *self, Subtree *tree, SubtreeArray *children) { self->scratch_tree = *tree; self->scratch_tree.children.size = 0; ts_subtree_set_children(&self->scratch_tree, children, self->language); - if (parser__select_tree(self, tree, &self->scratch_tree)) { + if (ts_parser__select_tree(self, tree, &self->scratch_tree)) { *tree = self->scratch_tree; return true; } else { @@ -578,7 +579,7 @@ static bool parser__replace_children(Parser *self, Subtree *tree, SubtreeArray * } } -static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymbol symbol, +static StackSliceArray ts_parser__reduce(TSParser *self, StackVersion version, TSSymbol symbol, uint32_t count, int dynamic_precedence, uint16_t alias_sequence_id, bool fragile) { uint32_t initial_version_count = ts_stack_version_count(self->stack); @@ -614,7 +615,7 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb children.size--; } - if (parser__replace_children(self, parent, &children)) { + if (ts_parser__replace_children(self, parent, &children)) { ts_subtree_array_delete(&self->tree_pool, &slice.subtrees); slice = next_slice; } else { @@ -669,7 +670,7 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb return pop; } -static void parser__start(Parser *self, TSInput input, Subtree *previous_tree) { +static void ts_parser__start(TSParser *self, TSInput input, Subtree *previous_tree) { if (previous_tree) { LOG("parse_after_edit"); } else { @@ -688,7 +689,7 @@ static void parser__start(Parser *self, TSInput input, Subtree *previous_tree) { self->in_ambiguity = false; } -static void parser__accept(Parser *self, StackVersion version, Subtree *lookahead) { +static void ts_parser__accept(TSParser *self, StackVersion version, Subtree *lookahead) { lookahead->extra = true; assert(lookahead->symbol == ts_builtin_sym_end); ts_subtree_retain(lookahead); @@ -719,7 +720,7 @@ static void parser__accept(Parser *self, StackVersion version, Subtree *lookahea self->accept_count++; if (self->finished_tree) { - if (parser__select_tree(self, self->finished_tree, root)) { + if (ts_parser__select_tree(self, self->finished_tree, root)) { ts_subtree_release(&self->tree_pool, self->finished_tree); self->finished_tree = root; } else { @@ -734,7 +735,7 @@ static void parser__accept(Parser *self, StackVersion version, Subtree *lookahea ts_stack_halt(self->stack, version); } -static bool parser__do_all_potential_reductions(Parser *self, StackVersion starting_version, +static bool ts_parser__do_all_potential_reductions(TSParser *self, StackVersion starting_version, TSSymbol lookahead_symbol) { uint32_t initial_version_count = ts_stack_version_count(self->stack); @@ -793,7 +794,7 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start for (uint32_t i = 0; i < self->reduce_actions.size; i++) { ReduceAction action = self->reduce_actions.contents[i]; - parser__reduce( + ts_parser__reduce( self, version, action.symbol, action.count, action.dynamic_precedence, action.alias_sequence_id, true @@ -819,10 +820,11 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start return can_shift_lookahead_symbol; } -static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lookahead_symbol) { +static void ts_parser__handle_error(TSParser *self, StackVersion version, + TSSymbol lookahead_symbol) { // Perform any reductions that could have happened in this state, regardless of the lookahead. uint32_t previous_version_count = ts_stack_version_count(self->stack); - parser__do_all_potential_reductions(self, version, 0); + ts_parser__do_all_potential_reductions(self, version, 0); uint32_t version_count = ts_stack_version_count(self->stack); // Push a discontinuity onto the stack. Merge all of the stack versions that @@ -852,7 +854,7 @@ static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lo state_after_missing_symbol ); - if (parser__do_all_potential_reductions( + if (ts_parser__do_all_potential_reductions( self, version_with_missing_tree, lookahead_symbol )) { @@ -880,7 +882,7 @@ static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lo LOG_STACK(); } -static void parser__halt_parse(Parser *self) { +static void ts_parser__halt_parse(TSParser *self) { LOG("halting_parse"); LOG_STACK(); @@ -899,11 +901,11 @@ static void parser__halt_parse(Parser *self) { ts_stack_push(self->stack, 0, root_error, false, 0); Subtree *eof = ts_subtree_make_leaf(&self->tree_pool, ts_builtin_sym_end, length_zero(), length_zero(), self->language); - parser__accept(self, 0, eof); + ts_parser__accept(self, 0, eof); ts_subtree_release(&self->tree_pool, eof); } -static bool parser__recover_to_state(Parser *self, StackVersion version, unsigned depth, +static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, unsigned depth, TSStateId goal_state) { StackSliceArray pop = ts_stack_pop_count(self->stack, version, depth); StackVersion previous_version = STACK_VERSION_NONE; @@ -956,7 +958,7 @@ static bool parser__recover_to_state(Parser *self, StackVersion version, unsigne return previous_version != STACK_VERSION_NONE; } -static void parser__recover(Parser *self, StackVersion version, Subtree *lookahead) { +static void ts_parser__recover(TSParser *self, StackVersion version, Subtree *lookahead) { bool did_recover = false; unsigned previous_version_count = ts_stack_version_count(self->stack); Length position = ts_stack_position(self->stack, version); @@ -991,10 +993,10 @@ static void parser__recover(Parser *self, StackVersion version, Subtree *lookahe entry.depth * ERROR_COST_PER_SKIPPED_TREE + (position.bytes - entry.position.bytes) * ERROR_COST_PER_SKIPPED_CHAR + (position.extent.row - entry.position.extent.row) * ERROR_COST_PER_SKIPPED_LINE; - if (parser__better_version_exists(self, version, false, new_cost)) break; + if (ts_parser__better_version_exists(self, version, false, new_cost)) break; if (ts_language_has_actions(self->language, entry.state, lookahead->symbol)) { - if (parser__recover_to_state(self, version, depth, entry.state)) { + if (ts_parser__recover_to_state(self, version, depth, entry.state)) { did_recover = true; LOG("recover_to_previous state:%u, depth:%u", entry.state, depth); LOG_STACK(); @@ -1020,7 +1022,7 @@ static void parser__recover(Parser *self, StackVersion version, Subtree *lookahe SubtreeArray children = array_new(); Subtree *parent = ts_subtree_make_error_node(&self->tree_pool, &children, self->language); ts_stack_push(self->stack, version, parent, false, 1); - parser__accept(self, version, lookahead); + ts_parser__accept(self, version, lookahead); return; } @@ -1029,7 +1031,7 @@ static void parser__recover(Parser *self, StackVersion version, Subtree *lookahe ts_subtree_total_bytes(lookahead) * ERROR_COST_PER_SKIPPED_CHAR + ts_subtree_total_size(lookahead).extent.row * ERROR_COST_PER_SKIPPED_LINE; - if (parser__better_version_exists(self, version, false, new_cost)) { + if (ts_parser__better_version_exists(self, version, false, new_cost)) { ts_stack_halt(self->stack, version); return; } @@ -1077,10 +1079,10 @@ static void parser__recover(Parser *self, StackVersion version, Subtree *lookahe } } -static void parser__advance(Parser *self, StackVersion version, ReusableNode *reusable_node) { +static void ts_parser__advance(TSParser *self, StackVersion version, ReusableNode *reusable_node) { TSStateId state = ts_stack_state(self->stack, version); TableEntry table_entry; - Subtree *lookahead = parser__get_lookahead(self, version, &state, reusable_node, &table_entry); + Subtree *lookahead = ts_parser__get_lookahead(self, version, &state, reusable_node, &table_entry); for (;;) { StackVersion last_reduction_version = STACK_VERSION_NONE; @@ -1105,11 +1107,11 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re } if (lookahead->children.size > 0) { - parser__breakdown_lookahead(self, &lookahead, state, reusable_node); + ts_parser__breakdown_lookahead(self, &lookahead, state, reusable_node); next_state = ts_language_next_state(self->language, state, lookahead->symbol); } - parser__shift(self, version, next_state, lookahead, action.params.extra); + ts_parser__shift(self, version, next_state, lookahead, action.params.extra); if (lookahead == reusable_node_tree(reusable_node)) { reusable_node_advance(reusable_node); } @@ -1120,7 +1122,7 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re case TSParseActionTypeReduce: { bool is_fragile = table_entry.action_count > 1; LOG("reduce sym:%s, child_count:%u", SYM_NAME(action.params.symbol), action.params.child_count); - StackSliceArray reduction = parser__reduce( + StackSliceArray reduction = ts_parser__reduce( self, version, action.params.symbol, action.params.child_count, action.params.dynamic_precedence, action.params.alias_sequence_id, is_fragile @@ -1132,16 +1134,16 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re case TSParseActionTypeAccept: { LOG("accept"); - parser__accept(self, version, lookahead); + ts_parser__accept(self, version, lookahead); ts_subtree_release(&self->tree_pool, lookahead); return; } case TSParseActionTypeRecover: { while (lookahead->children.size > 0) { - parser__breakdown_lookahead(self, &lookahead, state, reusable_node); + ts_parser__breakdown_lookahead(self, &lookahead, state, reusable_node); } - parser__recover(self, version, lookahead); + ts_parser__recover(self, version, lookahead); if (lookahead == reusable_node_tree(reusable_node)) { reusable_node_advance(reusable_node); } @@ -1155,10 +1157,10 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re ts_stack_renumber_version(self->stack, last_reduction_version, version); LOG_STACK(); } else if (state == ERROR_STATE) { - parser__recover(self, version, lookahead); + ts_parser__recover(self, version, lookahead); ts_subtree_release(&self->tree_pool, lookahead); return; - } else if (!parser__breakdown_top_of_stack(self, version)) { + } else if (!ts_parser__breakdown_top_of_stack(self, version)) { LOG("detect_error"); ts_stack_pause(self->stack, version, lookahead->first_leaf.symbol); ts_subtree_release(&self->tree_pool, lookahead); @@ -1170,7 +1172,7 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re } } -static unsigned parser__condense_stack(Parser *self) { +static unsigned ts_parser__condense_stack(TSParser *self) { bool made_changes = false; unsigned min_error_cost = UINT_MAX; for (StackVersion i = 0; i < ts_stack_version_count(self->stack); i++) { @@ -1180,15 +1182,15 @@ static unsigned parser__condense_stack(Parser *self) { continue; } - ErrorStatus status_i = parser__version_status(self, i); + ErrorStatus status_i = ts_parser__version_status(self, i); if (!status_i.is_in_error && status_i.cost < min_error_cost) { min_error_cost = status_i.cost; } for (StackVersion j = 0; j < i; j++) { - ErrorStatus status_j = parser__version_status(self, j); + ErrorStatus status_j = ts_parser__version_status(self, j); - switch (parser__compare_versions(self, status_j, status_i)) { + switch (ts_parser__compare_versions(self, status_j, status_i)) { case ErrorComparisonTakeLeft: made_changes = true; ts_stack_remove_version(self->stack, i); @@ -1235,7 +1237,7 @@ static unsigned parser__condense_stack(Parser *self) { LOG("resume version:%u", i); min_error_cost = ts_stack_error_cost(self->stack, i); TSSymbol lookahead_symbol = ts_stack_resume(self->stack, i); - parser__handle_error(self, i, lookahead_symbol); + ts_parser__handle_error(self, i, lookahead_symbol); has_unpaused_version = true; } else { ts_stack_remove_version(self->stack, i); @@ -1256,7 +1258,7 @@ static unsigned parser__condense_stack(Parser *self) { return min_error_cost; } -bool parser_init(Parser *self) { +bool ts_parser_init(TSParser *self) { ts_lexer_init(&self->lexer); array_init(&self->reduce_actions); array_reserve(&self->reduce_actions, 4); @@ -1264,11 +1266,11 @@ bool parser_init(Parser *self) { self->stack = ts_stack_new(&self->tree_pool); self->finished_tree = NULL; self->reusable_node = reusable_node_new(); - parser__set_cached_token(self, 0, NULL, NULL); + ts_parser__set_cached_token(self, 0, NULL, NULL); return true; } -void parser_set_language(Parser *self, const TSLanguage *language) { +void ts_parser_set_language(TSParser *self, const TSLanguage *language) { if (self->external_scanner_payload && self->language->external_scanner.destroy) self->language->external_scanner.destroy(self->external_scanner_payload); @@ -1280,18 +1282,18 @@ void parser_set_language(Parser *self, const TSLanguage *language) { self->language = language; } -void parser_destroy(Parser *self) { +void ts_parser_destroy(TSParser *self) { if (self->stack) ts_stack_delete(self->stack); if (self->reduce_actions.contents) array_delete(&self->reduce_actions); ts_subtree_pool_delete(&self->tree_pool); reusable_node_delete(&self->reusable_node); - parser_set_language(self, NULL); + ts_parser_set_language(self, NULL); } -Subtree *parser_parse(Parser *self, TSInput input, Subtree *old_tree, bool halt_on_error) { - parser__start(self, input, old_tree); +Subtree *ts_parser_parse(TSParser *self, TSInput input, Subtree *old_tree, bool halt_on_error) { + ts_parser__start(self, input, old_tree); StackVersion version = STACK_VERSION_NONE; uint32_t position = 0, last_position = 0; @@ -1309,7 +1311,7 @@ Subtree *parser_parse(Parser *self, TSInput input, Subtree *old_tree, bool halt_ ts_stack_position(self->stack, version).extent.row, ts_stack_position(self->stack, version).extent.column); - parser__advance(self, version, &reusable_node); + ts_parser__advance(self, version, &reusable_node); LOG_STACK(); position = ts_stack_position(self->stack, version).bytes; @@ -1322,11 +1324,11 @@ Subtree *parser_parse(Parser *self, TSInput input, Subtree *old_tree, bool halt_ reusable_node_assign(&self->reusable_node, &reusable_node); - unsigned min_error_cost = parser__condense_stack(self); + unsigned min_error_cost = ts_parser__condense_stack(self); if (self->finished_tree && self->finished_tree->error_cost < min_error_cost) { break; } else if (halt_on_error && min_error_cost > 0) { - parser__halt_parse(self); + ts_parser__halt_parse(self); break; } @@ -1335,7 +1337,7 @@ Subtree *parser_parse(Parser *self, TSInput input, Subtree *old_tree, bool halt_ reusable_node_delete(&reusable_node); ts_stack_clear(self->stack); - parser__set_cached_token(self, 0, NULL, NULL); + ts_parser__set_cached_token(self, 0, NULL, NULL); ts_subtree_balance(self->finished_tree, &self->tree_pool, self->language); LOG("done"); diff --git a/src/runtime/parser.h b/src/runtime/parser.h index ff776bc2..46cc7489 100644 --- a/src/runtime/parser.h +++ b/src/runtime/parser.h @@ -32,12 +32,12 @@ typedef struct { bool in_ambiguity; bool print_debugging_graphs; unsigned accept_count; -} Parser; +} TSParser; -bool parser_init(Parser *); -void parser_destroy(Parser *); -Subtree *parser_parse(Parser *, TSInput, Subtree *, bool halt_on_error); -void parser_set_language(Parser *, const TSLanguage *); +bool ts_parser_init(TSParser *); +void ts_parser_destroy(TSParser *); +Subtree *ts_parser_parse(TSParser *, TSInput, Subtree *, bool halt_on_error); +void ts_parser_set_language(TSParser *, const TSLanguage *); #ifdef __cplusplus } From e75ecd1bb125deea42ac1567b6ed442bcdf8bbfb Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 10 May 2018 22:22:37 -0700 Subject: [PATCH 20/31] Rework API completely --- include/tree_sitter/runtime.h | 49 ++- project.gyp | 2 +- src/runtime/document.c | 188 ---------- src/runtime/document.h | 20 -- src/runtime/get_changed_ranges.c | 34 +- src/runtime/node.c | 22 +- src/runtime/parser.c | 118 +++++-- src/runtime/parser.h | 46 --- src/runtime/reusable_node.h | 2 +- src/runtime/string_input.c | 25 +- src/runtime/string_input.h | 10 +- src/runtime/subtree.c | 17 +- src/runtime/subtree.h | 8 +- src/runtime/tree.c | 59 ++++ src/runtime/tree.h | 19 ++ src/runtime/tree_cursor.c | 32 +- src/runtime/tree_cursor.h | 6 +- test/benchmarks.cc | 39 +-- test/helpers/scope_sequence.cc | 11 +- test/helpers/scope_sequence.h | 2 +- test/integration/fuzzing-examples.cc | 20 +- test/integration/real_grammars.cc | 139 +++++--- test/integration/test_grammars.cc | 18 +- test/runtime/document_test.cc | 490 --------------------------- test/runtime/language_test.cc | 12 +- test/runtime/node_test.cc | 56 +-- test/runtime/parser_test.cc | 266 +++++++++++---- test/runtime/stack_test.cc | 2 +- test/runtime/subtree_test.cc | 2 +- test/runtime/tree_test.cc | 200 +++++++++++ tests.gyp | 2 +- 31 files changed, 841 insertions(+), 1075 deletions(-) delete mode 100644 src/runtime/document.c delete mode 100644 src/runtime/document.h delete mode 100644 src/runtime/parser.h create mode 100644 src/runtime/tree.c create mode 100644 src/runtime/tree.h delete mode 100644 test/runtime/document_test.cc create mode 100644 test/runtime/tree_test.cc diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index d3e282fb..fa6fd919 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -5,6 +5,7 @@ extern "C" { #endif +#include #include #include #include @@ -13,7 +14,8 @@ extern "C" { typedef unsigned short TSSymbol; typedef struct TSLanguage TSLanguage; -typedef struct TSDocument TSDocument; +typedef struct TSParser TSParser; +typedef struct TSTree TSTree; typedef struct TSTreeCursor TSTreeCursor; typedef enum { @@ -65,17 +67,29 @@ typedef struct { typedef struct { const void *subtree; - const TSDocument *document; + const TSTree *tree; TSPoint position; uint32_t byte; TSSymbol alias_symbol; } TSNode; -typedef struct { - TSRange **changed_ranges; - uint32_t *changed_range_count; - bool halt_on_error; -} TSParseOptions; +TSParser *ts_parser_new(); +void ts_parser_delete(TSParser *); +const TSLanguage *ts_parser_language(const TSParser *); +bool ts_parser_set_language(TSParser *, const TSLanguage *); +TSLogger ts_parser_logger(const TSParser *); +void ts_parser_set_logger(TSParser *, TSLogger); +void ts_parser_print_debugging_graphs(TSParser *, bool); +void ts_parser_halt_on_error(TSParser *, bool); +TSTree *ts_parser_parse(TSParser *, const TSTree *, TSInput); +TSTree *ts_parser_parse_string(TSParser *, const TSTree *, const char *, uint32_t); + +TSTree *ts_tree_copy(const TSTree *); +void ts_tree_delete(const TSTree *); +TSNode ts_tree_root_node(const TSTree *); +void ts_tree_edit(TSTree *, const TSInputEdit *); +TSRange *ts_tree_get_changed_ranges(const TSTree *, const TSTree *, uint32_t *); +void ts_tree_print_dot_graph(const TSTree *, FILE *); uint32_t ts_node_start_byte(TSNode); TSPoint ts_node_start_point(TSNode); @@ -105,26 +119,7 @@ TSNode ts_node_named_descendant_for_byte_range(TSNode, uint32_t, uint32_t); TSNode ts_node_descendant_for_point_range(TSNode, TSPoint, TSPoint); TSNode ts_node_named_descendant_for_point_range(TSNode, TSPoint, TSPoint); -TSDocument *ts_document_new(); -void ts_document_free(TSDocument *); -const TSLanguage *ts_document_language(TSDocument *); -void ts_document_set_language(TSDocument *, const TSLanguage *); -TSInput ts_document_input(TSDocument *); -void ts_document_set_input(TSDocument *, TSInput); -void ts_document_set_input_string(TSDocument *, const char *); -void ts_document_set_input_string_with_length(TSDocument *, const char *, uint32_t); -TSLogger ts_document_logger(const TSDocument *); -void ts_document_set_logger(TSDocument *, TSLogger); -void ts_document_print_debugging_graphs(TSDocument *, bool); -void ts_document_edit(TSDocument *, TSInputEdit); -void ts_document_parse(TSDocument *); -void ts_document_parse_and_get_changed_ranges(TSDocument *, TSRange **, uint32_t *); -void ts_document_parse_with_options(TSDocument *, TSParseOptions); -void ts_document_invalidate(TSDocument *); -TSNode ts_document_root_node(const TSDocument *); -TSTreeCursor *ts_document_tree_cursor(const TSDocument *); -uint32_t ts_document_parse_count(const TSDocument *); - +TSTreeCursor *ts_tree_cursor_new(const TSTree *); void ts_tree_cursor_delete(TSTreeCursor *); bool ts_tree_cursor_goto_first_child(TSTreeCursor *); bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *); diff --git a/project.gyp b/project.gyp index cfc1776b..11174724 100644 --- a/project.gyp +++ b/project.gyp @@ -87,7 +87,6 @@ 'externals/utf8proc', ], 'sources': [ - 'src/runtime/document.c', 'src/runtime/get_changed_ranges.c', 'src/runtime/language.c', 'src/runtime/lexer.c', @@ -96,6 +95,7 @@ 'src/runtime/parser.c', 'src/runtime/string_input.c', 'src/runtime/subtree.c', + 'src/runtime/tree.c', 'src/runtime/tree_cursor.c', 'src/runtime/utf16.c', 'externals/utf8proc/utf8proc.c', diff --git a/src/runtime/document.c b/src/runtime/document.c deleted file mode 100644 index 7d175b6a..00000000 --- a/src/runtime/document.c +++ /dev/null @@ -1,188 +0,0 @@ -#include "runtime/alloc.h" -#include "runtime/subtree.h" -#include "runtime/parser.h" -#include "runtime/string_input.h" -#include "runtime/document.h" -#include "runtime/tree_cursor.h" -#include "runtime/get_changed_ranges.h" - -#define LOG(...) \ - snprintf(self->parser.lexer.debug_buffer, TREE_SITTER_SERIALIZATION_BUFFER_SIZE, __VA_ARGS__); \ - self->parser.lexer.logger.log(self->parser.lexer.logger.payload, TSLogTypeLex, self->parser.lexer.debug_buffer); \ - -TSDocument *ts_document_new() { - TSDocument *self = ts_calloc(1, sizeof(TSDocument)); - ts_parser_init(&self->parser); - array_init(&self->cursor1.stack); - array_init(&self->cursor2.stack); - return self; -} - -void ts_document_free(TSDocument *self) { - if (self->tree) ts_subtree_release(&self->parser.tree_pool, self->tree); - if (self->cursor1.stack.contents) array_delete(&self->cursor1.stack); - if (self->cursor2.stack.contents) array_delete(&self->cursor2.stack); - ts_parser_destroy(&self->parser); - ts_document_set_input(self, (TSInput){ - NULL, - NULL, - NULL, - TSInputEncodingUTF8, - }); - ts_free(self); -} - -const TSLanguage *ts_document_language(TSDocument *self) { - return self->parser.language; -} - -void ts_document_set_language(TSDocument *self, const TSLanguage *language) { - if (language->version != TREE_SITTER_LANGUAGE_VERSION) return; - ts_document_invalidate(self); - ts_parser_set_language(&self->parser, language); - if (self->tree) { - ts_subtree_release(&self->parser.tree_pool, self->tree); - self->tree = NULL; - } -} - -TSLogger ts_document_logger(const TSDocument *self) { - return self->parser.lexer.logger; -} - -void ts_document_set_logger(TSDocument *self, TSLogger logger) { - self->parser.lexer.logger = logger; -} - -void ts_document_print_debugging_graphs(TSDocument *self, bool should_print) { - self->parser.print_debugging_graphs = should_print; -} - -TSInput ts_document_input(TSDocument *self) { - return self->input; -} - -void ts_document_set_input(TSDocument *self, TSInput input) { - if (self->owns_input) - ts_free(self->input.payload); - self->input = input; - self->owns_input = false; -} - -void ts_document_set_input_string(TSDocument *self, const char *text) { - ts_document_invalidate(self); - TSInput input = ts_string_input_make(text); - ts_document_set_input(self, input); - if (input.payload) { - self->owns_input = true; - } -} - -void ts_document_set_input_string_with_length(TSDocument *self, const char *text, uint32_t length) { - ts_document_invalidate(self); - TSInput input = ts_string_input_make_with_length(text, length); - ts_document_set_input(self, input); - if (input.payload) { - self->owns_input = true; - } -} - -void ts_document_edit(TSDocument *self, TSInputEdit edit) { - if (!self->tree) - return; - - uint32_t max_bytes = ts_subtree_total_bytes(self->tree); - if (edit.start_byte > max_bytes) - return; - if (edit.bytes_removed > max_bytes - edit.start_byte) - edit.bytes_removed = max_bytes - edit.start_byte; - - self->tree = ts_subtree_edit(self->tree, &edit, &self->parser.tree_pool); - - if (self->parser.print_debugging_graphs) { - ts_subtree_print_dot_graph(self->tree, self->parser.language, stderr); - } -} - -void ts_document_parse(TSDocument *self) { - ts_document_parse_with_options(self, (TSParseOptions){ - .halt_on_error = false, - .changed_ranges = NULL, - .changed_range_count = NULL, - }); -} - -void ts_document_parse_and_get_changed_ranges(TSDocument *self, TSRange **ranges, - uint32_t *range_count) { - ts_document_parse_with_options(self, (TSParseOptions){ - .halt_on_error = false, - .changed_ranges = ranges, - .changed_range_count = range_count, - }); -} - -void ts_document_parse_with_options(TSDocument *self, TSParseOptions options) { - if (options.changed_ranges && options.changed_range_count) { - *options.changed_ranges = NULL; - *options.changed_range_count = 0; - } - - if (!self->input.read || !self->parser.language) - return; - - Subtree *reusable_tree = self->valid ? self->tree : NULL; - if (reusable_tree && !reusable_tree->has_changes) - return; - - Subtree *tree = ts_parser_parse(&self->parser, self->input, reusable_tree, options.halt_on_error); - - if (self->tree) { - Subtree *old_tree = self->tree; - self->tree = tree; - - if (options.changed_ranges && options.changed_range_count) { - *options.changed_range_count = ts_subtree_get_changed_ranges( - old_tree, tree, &self->cursor1, &self->cursor2, - self->parser.language, options.changed_ranges - ); - - if (self->parser.lexer.logger.log) { - for (unsigned i = 0; i < *options.changed_range_count; i++) { - TSRange range = (*options.changed_ranges)[i]; - LOG( - "changed_range start:[%u %u], end:[%u %u]", - range.start.row, range.start.column, - range.end.row, range.end.column - ); - } - } - } - - ts_subtree_release(&self->parser.tree_pool, old_tree); - } - - self->tree = tree; - self->parse_count++; - self->valid = true; -} - -void ts_document_invalidate(TSDocument *self) { - self->valid = false; -} - -TSNode ts_document_root_node(const TSDocument *self) { - return (TSNode) { - .subtree = self->tree, - .document = self, - .position = {0, 0}, - .byte = 0, - }; -} - -uint32_t ts_document_parse_count(const TSDocument *self) { - return self->parse_count; -} - -TSTreeCursor *ts_document_tree_cursor(const TSDocument *self) { - return ts_tree_cursor_new(self); -} diff --git a/src/runtime/document.h b/src/runtime/document.h deleted file mode 100644 index a7476ebf..00000000 --- a/src/runtime/document.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef RUNTIME_DOCUMENT_H_ -#define RUNTIME_DOCUMENT_H_ - -#include "runtime/parser.h" -#include "runtime/subtree.h" -#include "runtime/tree_cursor.h" -#include - -struct TSDocument { - TSParser parser; - TSInput input; - Subtree *tree; - TSTreeCursor cursor1; - TSTreeCursor cursor2; - size_t parse_count; - bool valid; - bool owns_input; -}; - -#endif diff --git a/src/runtime/get_changed_ranges.c b/src/runtime/get_changed_ranges.c index 76e01f29..032fdaab 100644 --- a/src/runtime/get_changed_ranges.c +++ b/src/runtime/get_changed_ranges.c @@ -34,7 +34,7 @@ typedef struct { static Iterator iterator_new(TSTreeCursor *cursor, Subtree *tree, const TSLanguage *language) { array_clear(&cursor->stack); array_push(&cursor->stack, ((TreeCursorEntry){ - .tree = tree, + .subtree = tree, .position = length_zero(), .child_index = 0, .structural_child_index = 0, @@ -56,25 +56,25 @@ Length iterator_start_position(Iterator *self) { if (self->in_padding) { return entry.position; } else { - return length_add(entry.position, entry.tree->padding); + return length_add(entry.position, entry.subtree->padding); } } Length iterator_end_position(Iterator *self) { TreeCursorEntry entry = *array_back(&self->cursor.stack); - Length result = length_add(entry.position, entry.tree->padding); + Length result = length_add(entry.position, entry.subtree->padding); if (self->in_padding) { return result; } else { - return length_add(result, entry.tree->size); + return length_add(result, entry.subtree->size); } } static bool iterator_tree_is_visible(const Iterator *self) { TreeCursorEntry entry = *array_back(&self->cursor.stack); - if (entry.tree->visible) return true; + if (entry.subtree->visible) return true; if (self->cursor.stack.size > 1) { - Subtree *parent = self->cursor.stack.contents[self->cursor.stack.size - 2].tree; + Subtree *parent = self->cursor.stack.contents[self->cursor.stack.size - 2].subtree; const TSSymbol *alias_sequence = ts_language_alias_sequence(self->language, parent->alias_sequence_id); return alias_sequence && alias_sequence[entry.structural_child_index] != 0; } @@ -94,7 +94,7 @@ static void iterator_get_visible_state(const Iterator *self, Subtree **tree, TreeCursorEntry entry = self->cursor.stack.contents[i]; if (i > 0) { - Subtree *parent = self->cursor.stack.contents[i - 1].tree; + Subtree *parent = self->cursor.stack.contents[i - 1].subtree; const TSSymbol *alias_sequence = ts_language_alias_sequence( self->language, parent->alias_sequence_id @@ -104,8 +104,8 @@ static void iterator_get_visible_state(const Iterator *self, Subtree **tree, } } - if (entry.tree->visible || *alias_symbol) { - *tree = entry.tree; + if (entry.subtree->visible || *alias_symbol) { + *tree = entry.subtree; *start_byte = entry.position.bytes; break; } @@ -128,14 +128,14 @@ static bool iterator_descend(Iterator *self, uint32_t goal_position) { TreeCursorEntry entry = *array_back(&self->cursor.stack); Length position = entry.position; uint32_t structural_child_index = 0; - for (uint32_t i = 0; i < entry.tree->children.size; i++) { - Subtree *child = entry.tree->children.contents[i]; + for (uint32_t i = 0; i < entry.subtree->children.size; i++) { + Subtree *child = entry.subtree->children.contents[i]; Length child_left = length_add(position, child->padding); Length child_right = length_add(child_left, child->size); if (child_right.bytes > goal_position) { array_push(&self->cursor.stack, ((TreeCursorEntry){ - .tree = child, + .subtree = child, .position = position, .child_index = i, .structural_child_index = structural_child_index, @@ -178,16 +178,16 @@ static void iterator_advance(Iterator *self) { TreeCursorEntry entry = array_pop(&self->cursor.stack); if (iterator_done(self)) return; - Subtree *parent = array_back(&self->cursor.stack)->tree; + Subtree *parent = array_back(&self->cursor.stack)->subtree; uint32_t child_index = entry.child_index + 1; if (parent->children.size > child_index) { - Length position = length_add(entry.position, ts_subtree_total_size(entry.tree)); + Length position = length_add(entry.position, ts_subtree_total_size(entry.subtree)); uint32_t structural_child_index = entry.structural_child_index; - if (!entry.tree->extra) structural_child_index++; + if (!entry.subtree->extra) structural_child_index++; Subtree *next_child = parent->children.contents[child_index]; array_push(&self->cursor.stack, ((TreeCursorEntry){ - .tree = next_child, + .subtree = next_child, .position = position, .child_index = child_index, .structural_child_index = structural_child_index, @@ -250,7 +250,7 @@ static inline void iterator_print_state(Iterator *self) { TreeCursorEntry entry = *array_back(&self->cursor.stack); TSPoint start = iterator_start_position(self).extent; TSPoint end = iterator_end_position(self).extent; - const char *name = ts_language_symbol_name(self->language, entry.tree->symbol); + const char *name = ts_language_symbol_name(self->language, entry.subtree->symbol); printf( "(%-25s %s\t depth:%u [%u, %u] - [%u, %u])", name, self->in_padding ? "(p)" : " ", diff --git a/src/runtime/node.c b/src/runtime/node.c index 4df3400a..e260f13b 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -1,13 +1,13 @@ #include #include "runtime/subtree.h" -#include "runtime/document.h" +#include "runtime/tree.h" #include "runtime/language.h" // NodeChildIterator typedef struct { const Subtree *parent; - const TSDocument *document; + const TSTree *tree; Length position; uint32_t child_index; uint32_t structural_child_index; @@ -19,7 +19,7 @@ typedef struct { static inline TSNode ts_node__null() { return (TSNode) { .subtree = NULL, - .document = NULL, + .tree = NULL, .position = {0, 0}, .byte = 0, }; @@ -32,12 +32,12 @@ static inline const Subtree *ts_node__tree(TSNode self) { static inline NodeChildIterator ts_node_child_iterator_begin(const TSNode *node) { const Subtree *tree = ts_node__tree(*node); const TSSymbol *alias_sequence = ts_language_alias_sequence( - node->document->parser.language, + node->tree->language, tree->alias_sequence_id ); return (NodeChildIterator) { .parent = tree, - .document = node->document, + .tree = node->tree, .position = {node->byte, node->position}, .child_index = 0, .structural_child_index = 0, @@ -57,7 +57,7 @@ static inline bool ts_node_child_iterator_next(NodeChildIterator *self, TSNode * } *result = (TSNode) { .subtree = child, - .document = self->document, + .tree = self->tree, .position = self->position.extent, .byte = self->position.bytes, .alias_symbol = alias_symbol, @@ -77,7 +77,7 @@ static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { ( self.alias_symbol && ts_language_symbol_metadata( - self.document->parser.language, + self.tree->language, self.alias_symbol ).named ) @@ -343,11 +343,11 @@ TSSymbol ts_node_symbol(TSNode self) { } const char *ts_node_type(TSNode self) { - return ts_language_symbol_name(self.document->parser.language, ts_node_symbol(self)); + return ts_language_symbol_name(self.tree->language, ts_node_symbol(self)); } char *ts_node_string(TSNode self) { - return ts_subtree_string(ts_node__tree(self), self.document->parser.language, false); + return ts_subtree_string(ts_node__tree(self), self.tree->language, false); } bool ts_node_eq(TSNode self, TSNode other) { @@ -360,7 +360,7 @@ bool ts_node_eq(TSNode self, TSNode other) { bool ts_node_is_named(TSNode self) { const Subtree *tree = ts_node__tree(self); return self.alias_symbol - ? ts_language_symbol_metadata(self.document->parser.language, self.alias_symbol).named + ? ts_language_symbol_metadata(self.tree->language, self.alias_symbol).named : tree->named; } @@ -378,7 +378,7 @@ bool ts_node_has_error(TSNode self) { } TSNode ts_node_parent(TSNode self) { - TSNode node = ts_document_root_node(self.document); + TSNode node = ts_tree_root_node(self.tree); uint32_t end_byte = ts_node_end_byte(self); if (node.subtree == self.subtree) return ts_node__null(); diff --git a/src/runtime/parser.c b/src/runtime/parser.c index f3f6fd64..1c7da6b1 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -1,4 +1,3 @@ -#include "runtime/parser.h" #include #include #include @@ -10,8 +9,12 @@ #include "runtime/array.h" #include "runtime/language.h" #include "runtime/alloc.h" +#include "runtime/stack.h" +#include "runtime/reusable_node.h" #include "runtime/reduce_action.h" #include "runtime/error_costs.h" +#include "runtime/string_input.h" +#include "runtime/tree.h" #define LOG(...) \ if (self->lexer.logger.log || self->print_debugging_graphs) { \ @@ -37,6 +40,29 @@ static const unsigned MAX_VERSION_COUNT = 6; static const unsigned MAX_SUMMARY_DEPTH = 16; static const unsigned MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE; +typedef struct { + Subtree *token; + Subtree *last_external_token; + uint32_t byte_index; +} TokenCache; + +struct TSParser { + Lexer lexer; + Stack *stack; + SubtreePool tree_pool; + const TSLanguage *language; + ReduceActionSet reduce_actions; + Subtree *finished_tree; + Subtree scratch_tree; + TokenCache token_cache; + ReusableNode reusable_node; + void *external_scanner_payload; + bool in_ambiguity; + bool print_debugging_graphs; + bool halt_on_error; + unsigned accept_count; +}; + typedef struct { unsigned cost; unsigned node_count; @@ -52,6 +78,8 @@ typedef enum { ErrorComparisonTakeRight, } ErrorComparison; +// Parser - Private + static void ts_parser__log(TSParser *self) { if (self->lexer.logger.log) { self->lexer.logger.log( @@ -670,7 +698,7 @@ static StackSliceArray ts_parser__reduce(TSParser *self, StackVersion version, T return pop; } -static void ts_parser__start(TSParser *self, TSInput input, Subtree *previous_tree) { +static void ts_parser__start(TSParser *self, TSInput input, const Subtree *previous_tree) { if (previous_tree) { LOG("parse_after_edit"); } else { @@ -1258,42 +1286,76 @@ static unsigned ts_parser__condense_stack(TSParser *self) { return min_error_cost; } -bool ts_parser_init(TSParser *self) { +// Parser - Public + +TSParser *ts_parser_new() { + TSParser *self = ts_calloc(1, sizeof(TSParser)); ts_lexer_init(&self->lexer); array_init(&self->reduce_actions); array_reserve(&self->reduce_actions, 4); - ts_subtree_pool_init(&self->tree_pool); + self->tree_pool = ts_subtree_pool_new(32); self->stack = ts_stack_new(&self->tree_pool); self->finished_tree = NULL; self->reusable_node = reusable_node_new(); + self->print_debugging_graphs = false; + self->halt_on_error = false; ts_parser__set_cached_token(self, 0, NULL, NULL); - return true; + return self; } -void ts_parser_set_language(TSParser *self, const TSLanguage *language) { - if (self->external_scanner_payload && self->language->external_scanner.destroy) - self->language->external_scanner.destroy(self->external_scanner_payload); - - if (language && language->external_scanner.create) - self->external_scanner_payload = language->external_scanner.create(); - else - self->external_scanner_payload = NULL; - - self->language = language; -} - -void ts_parser_destroy(TSParser *self) { - if (self->stack) +void ts_parser_delete(TSParser *self) { + if (self->stack) { ts_stack_delete(self->stack); - if (self->reduce_actions.contents) + } + if (self->reduce_actions.contents) { array_delete(&self->reduce_actions); + } ts_subtree_pool_delete(&self->tree_pool); reusable_node_delete(&self->reusable_node); ts_parser_set_language(self, NULL); + ts_free(self); } -Subtree *ts_parser_parse(TSParser *self, TSInput input, Subtree *old_tree, bool halt_on_error) { - ts_parser__start(self, input, old_tree); +const TSLanguage *ts_parser_language(const TSParser *self) { + return self->language; +} + +bool ts_parser_set_language(TSParser *self, const TSLanguage *language) { + if (language && language->version != TREE_SITTER_LANGUAGE_VERSION) return false; + + if (self->external_scanner_payload && self->language->external_scanner.destroy) { + self->language->external_scanner.destroy(self->external_scanner_payload); + } + + if (language && language->external_scanner.create) { + self->external_scanner_payload = language->external_scanner.create(); + } else { + self->external_scanner_payload = NULL; + } + + self->language = language; + return true; +} + +TSLogger ts_parser_logger(const TSParser *self) { + return self->lexer.logger; +} + +void ts_parser_set_logger(TSParser *self, TSLogger logger) { + self->lexer.logger = logger; +} + +void ts_parser_print_debugging_graphs(TSParser *self, bool should_print) { + self->print_debugging_graphs = should_print; +} + +void ts_parser_halt_on_error(TSParser *self, bool should_halt_on_error) { + self->halt_on_error = should_halt_on_error; +} + +TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) { + if (!self->language) return NULL; + ts_parser__start(self, input, old_tree ? old_tree->root : NULL); StackVersion version = STACK_VERSION_NONE; uint32_t position = 0, last_position = 0; @@ -1327,7 +1389,7 @@ Subtree *ts_parser_parse(TSParser *self, TSInput input, Subtree *old_tree, bool unsigned min_error_cost = ts_parser__condense_stack(self); if (self->finished_tree && self->finished_tree->error_cost < min_error_cost) { break; - } else if (halt_on_error && min_error_cost > 0) { + } else if (self->halt_on_error && min_error_cost > 0) { ts_parser__halt_parse(self); break; } @@ -1342,5 +1404,13 @@ Subtree *ts_parser_parse(TSParser *self, TSInput input, Subtree *old_tree, bool LOG("done"); LOG_TREE(); - return self->finished_tree; + + return ts_tree_new(self->finished_tree, self->language); +} + +TSTree *ts_parser_parse_string(TSParser *self, const TSTree *old_tree, + const char *string, uint32_t length) { + TSStringInput input; + ts_string_input_init(&input, string, length); + return ts_parser_parse(self, old_tree, input.input); } diff --git a/src/runtime/parser.h b/src/runtime/parser.h deleted file mode 100644 index 46cc7489..00000000 --- a/src/runtime/parser.h +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef RUNTIME_PARSER_H_ -#define RUNTIME_PARSER_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "runtime/stack.h" -#include "runtime/array.h" -#include "runtime/lexer.h" -#include "runtime/reusable_node.h" -#include "runtime/reduce_action.h" -#include "runtime/subtree.h" - -typedef struct { - Subtree *token; - Subtree *last_external_token; - uint32_t byte_index; -} TokenCache; - -typedef struct { - Lexer lexer; - Stack *stack; - SubtreePool tree_pool; - const TSLanguage *language; - ReduceActionSet reduce_actions; - Subtree *finished_tree; - Subtree scratch_tree; - TokenCache token_cache; - ReusableNode reusable_node; - void *external_scanner_payload; - bool in_ambiguity; - bool print_debugging_graphs; - unsigned accept_count; -} TSParser; - -bool ts_parser_init(TSParser *); -void ts_parser_destroy(TSParser *); -Subtree *ts_parser_parse(TSParser *, TSInput, Subtree *, bool halt_on_error); -void ts_parser_set_language(TSParser *, const TSLanguage *); - -#ifdef __cplusplus -} -#endif - -#endif // RUNTIME_PARSER_H_ diff --git a/src/runtime/reusable_node.h b/src/runtime/reusable_node.h index 5f1884d5..c1d4f06b 100644 --- a/src/runtime/reusable_node.h +++ b/src/runtime/reusable_node.h @@ -24,7 +24,7 @@ static inline void reusable_node_reset(ReusableNode *self, Subtree *tree) { })); } -static inline Subtree *reusable_node_tree(ReusableNode *self) { +static inline const Subtree *reusable_node_tree(ReusableNode *self) { return self->stack.size > 0 ? self->stack.contents[self->stack.size - 1].tree : NULL; diff --git a/src/runtime/string_input.c b/src/runtime/string_input.c index 53f69ee5..c4e13e0c 100644 --- a/src/runtime/string_input.c +++ b/src/runtime/string_input.c @@ -1,13 +1,7 @@ +#include "tree_sitter/runtime.h" #include "runtime/string_input.h" -#include "runtime/alloc.h" #include -typedef struct { - const char *string; - uint32_t position; - uint32_t length; -} TSStringInput; - static const char *ts_string_input__read(void *payload, uint32_t *bytes_read) { TSStringInput *input = (TSStringInput *)payload; if (input->position >= input->length) { @@ -26,17 +20,12 @@ static int ts_string_input__seek(void *payload, uint32_t byte, TSPoint _) { return (byte < input->length); } -TSInput ts_string_input_make(const char *string) { - return ts_string_input_make_with_length(string, strlen(string)); -} - -TSInput ts_string_input_make_with_length(const char *string, uint32_t length) { - TSStringInput *input = ts_malloc(sizeof(TSStringInput)); - input->string = string; - input->position = 0; - input->length = length; - return (TSInput){ - .payload = input, +void ts_string_input_init(TSStringInput *self, const char *string, uint32_t length) { + self->string = string; + self->position = 0; + self->length = length; + self->input = (TSInput) { + .payload = self, .read = ts_string_input__read, .seek = ts_string_input__seek, .encoding = TSInputEncodingUTF8, diff --git a/src/runtime/string_input.h b/src/runtime/string_input.h index c96cd416..19171e4f 100644 --- a/src/runtime/string_input.h +++ b/src/runtime/string_input.h @@ -7,8 +7,14 @@ extern "C" { #include "tree_sitter/runtime.h" -TSInput ts_string_input_make(const char *); -TSInput ts_string_input_make_with_length(const char *, uint32_t); +typedef struct { + const char *string; + uint32_t position; + uint32_t length; + TSInput input; +} TSStringInput; + +void ts_string_input_init(TSStringInput *, const char *, uint32_t); #ifdef __cplusplus } diff --git a/src/runtime/subtree.c b/src/runtime/subtree.c index 1808ee6d..aff2a6fc 100644 --- a/src/runtime/subtree.c +++ b/src/runtime/subtree.c @@ -19,6 +19,10 @@ typedef struct { TSStateId TS_TREE_STATE_NONE = USHRT_MAX; +static const uint32_t MAX_TREE_POOL_SIZE = 1024; + +static const TSExternalTokenState empty_state = {.length = 0, .short_data = {0}}; + // ExternalTokenState void ts_external_token_state_init(TSExternalTokenState *self, const char *content, unsigned length) { @@ -102,11 +106,10 @@ void ts_subtree_array_reverse(SubtreeArray *self) { // SubtreePool -static const uint32_t MAX_TREE_POOL_SIZE = 1024; - -void ts_subtree_pool_init(SubtreePool *self) { - array_init(&self->free_trees); - array_init(&self->tree_stack); +SubtreePool ts_subtree_pool_new(uint32_t capacity) { + SubtreePool self = {array_new(), array_new()}; + array_reserve(&self.free_trees, capacity); + return self; } void ts_subtree_pool_delete(SubtreePool *self) { @@ -128,7 +131,7 @@ Subtree *ts_subtree_pool_allocate(SubtreePool *self) { } void ts_subtree_pool_free(SubtreePool *self, Subtree *tree) { - if (self->free_trees.size < MAX_TREE_POOL_SIZE) { + if (self->free_trees.capacity > 0 && self->free_trees.size < MAX_TREE_POOL_SIZE) { array_push(&self->free_trees, tree); } else { ts_free(tree); @@ -691,8 +694,6 @@ void ts_subtree_print_dot_graph(const Subtree *self, const TSLanguage *language, fprintf(f, "}\n"); } -static const TSExternalTokenState empty_state = {.length = 0, .short_data = {0}}; - bool ts_subtree_external_token_state_eq(const Subtree *self, const Subtree *other) { const TSExternalTokenState *state1 = &empty_state; const TSExternalTokenState *state2 = &empty_state; diff --git a/src/runtime/subtree.h b/src/runtime/subtree.h index 5168ac38..fd1fcaff 100644 --- a/src/runtime/subtree.h +++ b/src/runtime/subtree.h @@ -1,5 +1,5 @@ -#ifndef RUNTIME_TREE_H_ -#define RUNTIME_TREE_H_ +#ifndef RUNTIME_SUBTREE_H_ +#define RUNTIME_SUBTREE_H_ #ifdef __cplusplus extern "C" { @@ -83,7 +83,7 @@ void ts_subtree_array_delete(SubtreePool *, SubtreeArray *); SubtreeArray ts_subtree_array_remove_trailing_extras(SubtreeArray *); void ts_subtree_array_reverse(SubtreeArray *); -void ts_subtree_pool_init(SubtreePool *); +SubtreePool ts_subtree_pool_new(uint32_t capacity); void ts_subtree_pool_delete(SubtreePool *); Subtree *ts_subtree_pool_allocate(SubtreePool *); void ts_subtree_pool_free(SubtreePool *, Subtree *); @@ -122,4 +122,4 @@ static inline Length ts_subtree_total_size(const Subtree *self) { } #endif -#endif // RUNTIME_TREE_H_ +#endif // RUNTIME_SUBTREE_H_ diff --git a/src/runtime/tree.c b/src/runtime/tree.c new file mode 100644 index 00000000..2c365644 --- /dev/null +++ b/src/runtime/tree.c @@ -0,0 +1,59 @@ +#include "tree_sitter/runtime.h" +#include "runtime/array.h" +#include "runtime/get_changed_ranges.h" +#include "runtime/subtree.h" +#include "runtime/tree_cursor.h" +#include "runtime/tree.h" + +TSTree *ts_tree_new(const Subtree *root, const TSLanguage *language) { + TSTree *result = ts_malloc(sizeof(TSTree)); + result->root = root; + result->language = language; + return result; +} + +TSTree *ts_tree_copy(const TSTree *self) { + ts_subtree_retain(self->root); + return ts_tree_new(self->root, self->language); +} + +void ts_tree_delete(const TSTree *self) { + SubtreePool pool = ts_subtree_pool_new(0); + ts_subtree_release(&pool, self->root); + ts_subtree_pool_delete(&pool); + ts_free(self); +} + +TSNode ts_tree_root_node(const TSTree *self) { + return (TSNode) { + .subtree = self->root, + .tree = self, + .position = {0, 0}, + .byte = 0, + .alias_symbol = 0, + }; +} + +void ts_tree_edit(TSTree *self, const TSInputEdit *edit) { + SubtreePool pool = ts_subtree_pool_new(0); + self->root = ts_subtree_edit(self->root, edit, &pool); + assert(pool.tree_stack.capacity == 0 && pool.free_trees.capacity == 0); +} + +TSRange *ts_tree_get_changed_ranges(const TSTree *self, const TSTree *other, uint32_t *count) { + TSRange *result; + TSTreeCursor cursor1, cursor2; + ts_tree_cursor_init(&cursor1, self); + ts_tree_cursor_init(&cursor2, self); + *count = ts_subtree_get_changed_ranges( + self->root, other->root, &cursor1, &cursor2, + self->language, &result + ); + array_delete(&cursor1.stack); + array_delete(&cursor2.stack); + return result; +} + +void ts_tree_print_dot_graph(const TSTree *self, FILE *file) { + ts_subtree_print_dot_graph(self->root, self->language, file); +} diff --git a/src/runtime/tree.h b/src/runtime/tree.h new file mode 100644 index 00000000..7429e06c --- /dev/null +++ b/src/runtime/tree.h @@ -0,0 +1,19 @@ +#ifndef RUNTIME_TREE_H_ +#define RUNTIME_TREE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +struct TSTree { + const Subtree *root; + const TSLanguage *language; +}; + +TSTree *ts_tree_new(const Subtree *root, const TSLanguage *language); + +#ifdef __cplusplus +} +#endif + +#endif // RUNTIME_TREE_H_ diff --git a/src/runtime/tree_cursor.c b/src/runtime/tree_cursor.c index 730dacf9..c7aa9691 100644 --- a/src/runtime/tree_cursor.c +++ b/src/runtime/tree_cursor.c @@ -1,20 +1,24 @@ #include "tree_sitter/runtime.h" #include "runtime/alloc.h" #include "runtime/tree_cursor.h" -#include "runtime/document.h" #include "runtime/language.h" +#include "runtime/tree.h" -TSTreeCursor *ts_tree_cursor_new(const TSDocument *document) { +TSTreeCursor *ts_tree_cursor_new(const TSTree *tree) { TSTreeCursor *self = ts_malloc(sizeof(TSTreeCursor)); - self->document = document; + ts_tree_cursor_init(self, tree); + return self; +} + +void ts_tree_cursor_init(TSTreeCursor *self, const TSTree *tree) { + self->tree = tree; array_init(&self->stack); array_push(&self->stack, ((TreeCursorEntry) { - .tree = document->tree, + .subtree = tree->root, .position = length_zero(), .child_index = 0, .structural_child_index = 0, })); - return self; } void ts_tree_cursor_delete(TSTreeCursor *self) { @@ -24,7 +28,7 @@ void ts_tree_cursor_delete(TSTreeCursor *self) { bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { TreeCursorEntry *last_entry = array_back(&self->stack); - Subtree *tree = last_entry->tree; + Subtree *tree = last_entry->subtree; Length position = last_entry->position; bool did_descend; @@ -36,7 +40,7 @@ bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { Subtree *child = tree->children.contents[i]; if (child->visible || child->visible_child_count > 0) { array_push(&self->stack, ((TreeCursorEntry) { - .tree = child, + .subtree = child, .child_index = i, .structural_child_index = structural_child_index, .position = position, @@ -64,7 +68,7 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) { TreeCursorEntry *parent_entry = &self->stack.contents[i]; - Subtree *parent = parent_entry->tree; + Subtree *parent = parent_entry->subtree; uint32_t child_index = child_entry->child_index; uint32_t structural_child_index = child_entry->structural_child_index; Length position = child_entry->position; @@ -77,7 +81,7 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { if (child->visible || child->visible_child_count > 0) { self->stack.contents[i + 1] = (TreeCursorEntry) { - .tree = child, + .subtree = child, .child_index = child_index, .structural_child_index = structural_child_index, .position = position, @@ -103,7 +107,7 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { bool ts_tree_cursor_goto_parent(TSTreeCursor *self) { for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) { TreeCursorEntry *entry = &self->stack.contents[i]; - if (entry->tree->visible) { + if (entry->subtree->visible) { self->stack.size = i + 1; return true; } @@ -117,16 +121,16 @@ TSNode ts_tree_cursor_current_node(TSTreeCursor *self) { if (self->stack.size > 1) { TreeCursorEntry *parent_entry = &self->stack.contents[self->stack.size - 2]; const TSSymbol *alias_sequence = ts_language_alias_sequence( - self->document->parser.language, - parent_entry->tree->alias_sequence_id + self->tree->language, + parent_entry->subtree->alias_sequence_id ); if (alias_sequence) { alias_symbol = alias_sequence[last_entry->structural_child_index]; } } return (TSNode) { - .document = self->document, - .subtree = last_entry->tree, + .tree = self->tree, + .subtree = last_entry->subtree, .position = last_entry->position.extent, .byte = last_entry->position.bytes, .alias_symbol = alias_symbol, diff --git a/src/runtime/tree_cursor.h b/src/runtime/tree_cursor.h index 5e571fba..c016b276 100644 --- a/src/runtime/tree_cursor.h +++ b/src/runtime/tree_cursor.h @@ -4,17 +4,17 @@ #include "runtime/subtree.h" typedef struct { - Subtree *tree; + Subtree *subtree; Length position; uint32_t child_index; uint32_t structural_child_index; } TreeCursorEntry; struct TSTreeCursor { - const TSDocument *document; + const TSTree *tree; Array(TreeCursorEntry) stack; }; -TSTreeCursor *ts_tree_cursor_new(const TSDocument *); +void ts_tree_cursor_init(TSTreeCursor *, const TSTree *); #endif // RUNTIME_TREE_CURSOR_H_ diff --git a/test/benchmarks.cc b/test/benchmarks.cc index 616d2de6..d4f475b5 100644 --- a/test/benchmarks.cc +++ b/test/benchmarks.cc @@ -43,12 +43,12 @@ int main(int argc, char *arg[]) { vector error_speeds; vector non_error_speeds; - auto document = ts_document_new(); + TSParser *parser = ts_parser_new(); if (getenv("TREE_SITTER_BENCHMARK_SVG")) { - ts_document_print_debugging_graphs(document, true); + ts_parser_print_debugging_graphs(parser, true); } else if (getenv("TREE_SITTER_BENCHMARK_LOG")) { - ts_document_set_logger(document, stderr_logger_new(false)); + ts_parser_set_logger(parser, stderr_logger_new(false)); } auto language_filter = getenv("TREE_SITTER_BENCHMARK_LANGUAGE"); @@ -61,7 +61,7 @@ int main(int argc, char *arg[]) { for (auto &language_name : language_names) { if (language_filter && language_name != language_filter) continue; - ts_document_set_language(document, load_real_language(language_name)); + ts_parser_set_language(parser, load_real_language(language_name)); printf("%s\n", language_name.c_str()); @@ -69,20 +69,16 @@ int main(int argc, char *arg[]) { if (file_name_filter && example.file_name != file_name_filter) continue; if (example.input.size() < 256) continue; - ts_document_invalidate(document); - ts_document_set_input_string(document, ""); - ts_document_parse(document); - - ts_document_invalidate(document); - ts_document_set_input_string(document, example.input.c_str()); - clock_t start_time = clock(); - ts_document_parse(document); + TSTree *tree = ts_parser_parse_string(parser, nullptr, example.input.c_str(), example.input.size()); clock_t end_time = clock(); - unsigned duration = (end_time - start_time) * 1000 / CLOCKS_PER_SEC; - assert(!ts_node_has_error(ts_document_root_node(document))); + + assert(!ts_node_has_error(ts_tree_root_node(tree))); + ts_tree_delete(tree); + + size_t duration = (end_time - start_time) * 1000 / CLOCKS_PER_SEC; size_t speed = static_cast(example.input.size()) / duration; - printf(" %-30s\t%u ms\t\t%lu bytes/ms\n", example.file_name.c_str(), duration, speed); + printf(" %-30s\t%lu ms\t\t%lu bytes/ms\n", example.file_name.c_str(), duration, speed); if (speed != 0) non_error_speeds.push_back(speed); } @@ -93,15 +89,15 @@ int main(int argc, char *arg[]) { if (file_name_filter && example.file_name != file_name_filter) continue; if (example.input.size() < 256) continue; - ts_document_invalidate(document); - ts_document_set_input_string(document, example.input.c_str()); - clock_t start_time = clock(); - ts_document_parse(document); + TSTree *tree = ts_parser_parse_string(parser, nullptr, example.input.c_str(), example.input.size()); clock_t end_time = clock(); - unsigned duration = (end_time - start_time) * 1000 / CLOCKS_PER_SEC; + + ts_tree_delete(tree); + + size_t duration = (end_time - start_time) * 1000 / CLOCKS_PER_SEC; size_t speed = static_cast(example.input.size()) / duration; - printf(" %-30s\t%u ms\t\t%lu bytes/ms\n", example.file_name.c_str(), duration, speed); + printf(" %-30s\t%lu ms\t\t%lu bytes/ms\n", example.file_name.c_str(), duration, speed); if (speed != 0) error_speeds.push_back(speed); } } @@ -118,5 +114,6 @@ int main(int argc, char *arg[]) { printf(" %-30s\t%lu bytes/ms\n", "average speed", mean(error_speeds)); printf(" %-30s\t%lu bytes/ms\n", "worst speed", min(error_speeds)); + ts_parser_delete(parser); return 0; } diff --git a/test/helpers/scope_sequence.cc b/test/helpers/scope_sequence.cc index c3db70ac..8851f0c4 100644 --- a/test/helpers/scope_sequence.cc +++ b/test/helpers/scope_sequence.cc @@ -21,8 +21,7 @@ static void append_text_to_scope_sequence(ScopeSequence *sequence, static void append_to_scope_sequence(ScopeSequence *sequence, ScopeStack *current_scopes, - TSNode node, TSDocument *document, - const std::string &text) { + TSNode node, const std::string &text) { append_text_to_scope_sequence( sequence, current_scopes, text, ts_node_start_byte(node) - sequence->size() ); @@ -31,7 +30,7 @@ static void append_to_scope_sequence(ScopeSequence *sequence, for (size_t i = 0, n = ts_node_child_count(node); i < n; i++) { TSNode child = ts_node_child(node, i); - append_to_scope_sequence(sequence, current_scopes, child, document, text); + append_to_scope_sequence(sequence, current_scopes, child, text); } append_text_to_scope_sequence( @@ -41,11 +40,11 @@ static void append_to_scope_sequence(ScopeSequence *sequence, current_scopes->pop_back(); } -ScopeSequence build_scope_sequence(TSDocument *document, const std::string &text) { +ScopeSequence build_scope_sequence(TSTree *tree, const std::string &text) { ScopeSequence sequence; ScopeStack current_scopes; - TSNode node = ts_document_root_node(document); - append_to_scope_sequence(&sequence, ¤t_scopes, node, document, text); + TSNode node = ts_tree_root_node(tree); + append_to_scope_sequence(&sequence, ¤t_scopes, node, text); return sequence; } diff --git a/test/helpers/scope_sequence.h b/test/helpers/scope_sequence.h index c83ad597..2ad15117 100644 --- a/test/helpers/scope_sequence.h +++ b/test/helpers/scope_sequence.h @@ -9,7 +9,7 @@ typedef std::string Scope; typedef std::vector ScopeStack; typedef std::vector ScopeSequence; -ScopeSequence build_scope_sequence(TSDocument *document, const std::string &text); +ScopeSequence build_scope_sequence(TSTree *tree, const std::string &text); void verify_changed_ranges(const ScopeSequence &old, const ScopeSequence &new_sequence, const std::string &text, TSRange *ranges, size_t range_count); diff --git a/test/integration/fuzzing-examples.cc b/test/integration/fuzzing-examples.cc index 46cac15a..92682d4c 100644 --- a/test/integration/fuzzing-examples.cc +++ b/test/integration/fuzzing-examples.cc @@ -29,14 +29,14 @@ describe("examples found via fuzzing", [&]() { for (unsigned i = 0, n = examples.size(); i < n; i++) { it(("parses example number " + to_string(i)).c_str(), [&]() { - TSDocument *document = ts_document_new(); + TSParser *parser = ts_parser_new(); if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) { - ts_document_print_debugging_graphs(document, true); + ts_parser_print_debugging_graphs(parser, true); } const string &language_name = examples[i].first; - ts_document_set_language(document, load_real_language(language_name)); + ts_parser_set_language(parser, load_real_language(language_name)); string input; const string &base64_input = examples[i].second; @@ -47,18 +47,12 @@ describe("examples found via fuzzing", [&]() { base64_input.size() )); - ts_document_set_input_string_with_length( - document, - input.c_str(), - input.size() - ); - - ts_document_parse(document); - - TSNode node = ts_document_root_node(document); + TSTree *tree = ts_parser_parse_string(parser, nullptr, input.c_str(), input.size()); + TSNode node = ts_tree_root_node(tree); assert_consistent_tree_sizes(node); - ts_document_free(document); + ts_tree_delete(tree); + ts_parser_delete(parser); }); } diff --git a/test/integration/real_grammars.cc b/test/integration/real_grammars.cc index 608ded82..0d594ce5 100644 --- a/test/integration/real_grammars.cc +++ b/test/integration/real_grammars.cc @@ -12,8 +12,8 @@ #include "helpers/tree_helpers.h" #include -static void assert_correct_tree_size(TSDocument *document, string content) { - TSNode root_node = ts_document_root_node(document); +static void assert_correct_tree_size(TSTree *tree, string content) { + TSNode root_node = ts_tree_root_node(tree); AssertThat(ts_node_end_byte(root_node), Equals(content.size())); assert_consistent_tree_sizes(root_node); } @@ -33,48 +33,43 @@ vector test_languages({ for (auto &language_name : test_languages) { describe(("the " + language_name + " language").c_str(), [&]() { - TSDocument *document; + TSParser *parser; const bool debug_graphs_enabled = getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS"); before_each([&]() { record_alloc::start(); - document = ts_document_new(); - ts_document_set_language(document, load_real_language(language_name)); + parser = ts_parser_new(); + ts_parser_set_language(parser, load_real_language(language_name)); - // ts_document_set_logger(document, stderr_logger_new(true)); + // ts_parser_set_logger(parser, stderr_logger_new(true)); if (debug_graphs_enabled) { - ts_document_print_debugging_graphs(document, true); + ts_parser_print_debugging_graphs(parser, true); } }); after_each([&]() { - ts_document_free(document); + ts_parser_delete(parser); AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); }); for (auto &entry : read_real_language_corpus(language_name)) { SpyInput *input; - auto it_handles_edit_sequence = [&](string name, std::function edit_sequence){ - it(("parses " + entry.description + ": " + name).c_str(), [&]() { - input = new SpyInput(entry.input, 3); - if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); - ts_document_set_input(document, input->input()); - edit_sequence(); + it(("parses " + entry.description + ": initial parse").c_str(), [&]() { + input = new SpyInput(entry.input, 3); + if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); - TSNode root_node = ts_document_root_node(document); - const char *node_string = ts_node_string(root_node); - string result(node_string); - ts_free((void *)node_string); - AssertThat(result, Equals(entry.tree_string)); + TSTree *tree = ts_parser_parse(parser, nullptr, input->input()); + assert_correct_tree_size(tree, input->content); - assert_correct_tree_size(document, input->content); - delete input; - }); - }; + TSNode root_node = ts_tree_root_node(tree); + const char *node_string = ts_node_string(root_node); + string result(node_string); + ts_free((void *)node_string); + AssertThat(result, Equals(entry.tree_string)); - it_handles_edit_sequence("initial parse", [&]() { - ts_document_parse(document); + ts_tree_delete(tree); + delete input; }); set> deletions; @@ -86,54 +81,88 @@ for (auto &language_name : test_languages) { string inserted_text = random_words(random_unsigned(4) + 1); if (insertions.insert({edit_position, inserted_text}).second) { - string description = "\"" + inserted_text + "\" at " + to_string(edit_position); - - it_handles_edit_sequence("repairing an insertion of " + description, [&]() { - ts_document_edit(document, input->replace(edit_position, 0, inserted_text)); - ts_document_parse(document); - assert_correct_tree_size(document, input->content); + it(("parses " + entry.description + + ": repairing an insertion of \"" + inserted_text + "\"" + + " at " + to_string(edit_position)).c_str(), [&]() { + input = new SpyInput(entry.input, 3); if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); - ts_document_edit(document, input->undo()); - assert_correct_tree_size(document, input->content); + input->replace(edit_position, 0, inserted_text); + TSTree *tree = ts_parser_parse(parser, nullptr, input->input()); + assert_correct_tree_size(tree, input->content); if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); - TSRange *ranges; + TSInputEdit edit = input->undo(); + ts_tree_edit(tree, &edit); + assert_correct_tree_size(tree, input->content); + if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); + + TSTree *new_tree = ts_parser_parse(parser, tree, input->input()); + assert_correct_tree_size(new_tree, input->content); + uint32_t range_count; - ScopeSequence old_scope_sequence = build_scope_sequence(document, input->content); - ts_document_parse_and_get_changed_ranges(document, &ranges, &range_count); - assert_correct_tree_size(document, input->content); + TSRange *ranges = ts_tree_get_changed_ranges(tree, new_tree, &range_count); - ScopeSequence new_scope_sequence = build_scope_sequence(document, input->content); - verify_changed_ranges(old_scope_sequence, new_scope_sequence, - input->content, ranges, range_count); + ScopeSequence old_scope_sequence = build_scope_sequence(tree, input->content); + ScopeSequence new_scope_sequence = build_scope_sequence(new_tree, input->content); + verify_changed_ranges( + old_scope_sequence, new_scope_sequence, + input->content, ranges, range_count + ); ts_free(ranges); + + TSNode root_node = ts_tree_root_node(new_tree); + const char *node_string = ts_node_string(root_node); + string result(node_string); + ts_free((void *)node_string); + AssertThat(result, Equals(entry.tree_string)); + + ts_tree_delete(tree); + ts_tree_delete(new_tree); + delete input; }); } if (deletions.insert({edit_position, deletion_size}).second) { - string desription = to_string(edit_position) + "-" + to_string(edit_position + deletion_size); - - it_handles_edit_sequence("repairing a deletion of " + desription, [&]() { - ts_document_edit(document, input->replace(edit_position, deletion_size, "")); - ts_document_parse(document); - assert_correct_tree_size(document, input->content); + it(("parses " + entry.description + + ": repairing a deletion of " + + to_string(edit_position) + "-" + to_string(edit_position + deletion_size)).c_str(), [&]() { + input = new SpyInput(entry.input, 3); if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); - ts_document_edit(document, input->undo()); - assert_correct_tree_size(document, input->content); + input->replace(edit_position, deletion_size, ""); + TSTree *tree = ts_parser_parse(parser, nullptr, input->input()); + assert_correct_tree_size(tree, input->content); if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); - TSRange *ranges; + TSInputEdit edit = input->undo(); + ts_tree_edit(tree, &edit); + assert_correct_tree_size(tree, input->content); + if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); + + TSTree *new_tree = ts_parser_parse(parser, tree, input->input()); + assert_correct_tree_size(new_tree, input->content); + uint32_t range_count; - ScopeSequence old_scope_sequence = build_scope_sequence(document, input->content); - ts_document_parse_and_get_changed_ranges(document, &ranges, &range_count); - assert_correct_tree_size(document, input->content); + TSRange *ranges = ts_tree_get_changed_ranges(tree, new_tree, &range_count); - ScopeSequence new_scope_sequence = build_scope_sequence(document, input->content); - verify_changed_ranges(old_scope_sequence, new_scope_sequence, - input->content, ranges, range_count); + ScopeSequence old_scope_sequence = build_scope_sequence(tree, input->content); + ScopeSequence new_scope_sequence = build_scope_sequence(new_tree, input->content); + verify_changed_ranges( + old_scope_sequence, new_scope_sequence, + input->content, ranges, range_count + ); ts_free(ranges); + + TSNode root_node = ts_tree_root_node(new_tree); + const char *node_string = ts_node_string(root_node); + string result(node_string); + ts_free((void *)node_string); + AssertThat(result, Equals(entry.tree_string)); + + ts_tree_delete(tree); + ts_tree_delete(new_tree); + delete input; }); } } diff --git a/test/integration/test_grammars.cc b/test/integration/test_grammars.cc index f5324579..bba40ace 100644 --- a/test/integration/test_grammars.cc +++ b/test/integration/test_grammars.cc @@ -52,26 +52,26 @@ for (auto &language_name : test_languages) { ); } - TSDocument *document = ts_document_new(); - ts_document_set_language(document, language); - ts_document_set_input_string_with_length(document, entry.input.c_str(), entry.input.size()); + TSParser *parser = ts_parser_new(); + ts_parser_set_language(parser, language); - // ts_document_print_debugging_graphs(document, true); if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) { - ts_document_print_debugging_graphs(document, true); + ts_parser_print_debugging_graphs(parser, true); } - ts_document_parse(document); + TSTree *tree = ts_parser_parse_string(parser, nullptr, entry.input.c_str(), entry.input.size()); - TSNode root_node = ts_document_root_node(document); + TSNode root_node = ts_tree_root_node(tree); AssertThat(ts_node_end_byte(root_node), Equals(entry.input.size())); assert_consistent_tree_sizes(root_node); + const char *node_string = ts_node_string(root_node); string result(node_string); ts_free((void *)node_string); - ts_document_free(document); - AssertThat(result, Equals(entry.tree_string)); + + ts_tree_delete(tree); + ts_parser_delete(parser); AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); }); } diff --git a/test/runtime/document_test.cc b/test/runtime/document_test.cc deleted file mode 100644 index 5f140251..00000000 --- a/test/runtime/document_test.cc +++ /dev/null @@ -1,490 +0,0 @@ -#include "test_helper.h" -#include "runtime/alloc.h" -#include "helpers/record_alloc.h" -#include "helpers/stream_methods.h" -#include "helpers/tree_helpers.h" -#include "helpers/point_helpers.h" -#include "helpers/spy_logger.h" -#include "helpers/stderr_logger.h" -#include "helpers/spy_input.h" -#include "helpers/load_language.h" - -TSPoint point(size_t row, size_t column) { - return TSPoint{static_cast(row), static_cast(column)}; -} - -START_TEST - -describe("Document", [&]() { - TSDocument *document; - TSNode root; - - before_each([&]() { - record_alloc::start(); - document = ts_document_new(); - - if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) { - ts_document_print_debugging_graphs(document, true); - } - }); - - after_each([&]() { - ts_document_free(document); - record_alloc::stop(); - AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); - }); - - auto assert_node_string_equals = [&](TSNode node, const string &expected) { - char *str = ts_node_string(node); - string actual(str); - ts_free(str); - AssertThat(actual, Equals(expected)); - }; - - describe("set_input(input)", [&]() { - SpyInput *spy_input; - - before_each([&]() { - spy_input = new SpyInput("{\"key\": [null, 2]}", 3); - - ts_document_set_language(document, load_real_language("json")); - ts_document_set_input_string(document, "{\"key\": [1, 2]}"); - ts_document_parse(document); - - root = ts_document_root_node(document); - assert_node_string_equals( - root, - "(value (object (pair (string) (array (number) (number)))))"); - }); - - after_each([&]() { - delete spy_input; - }); - - it("handles both UTF8 and UTF16 encodings", [&]() { - const char16_t content[] = u"[true, false]"; - spy_input->content = string((const char *)content, sizeof(content)); - spy_input->encoding = TSInputEncodingUTF16; - - ts_document_set_input(document, spy_input->input()); - ts_document_invalidate(document); - ts_document_parse(document); - - root = ts_document_root_node(document); - assert_node_string_equals( - root, - "(value (array (true) (false)))"); - }); - - it("handles truncated UTF16 data", [&]() { - const char content[1] = { '\0' }; - spy_input->content = string(content, sizeof(content)); - spy_input->encoding = TSInputEncodingUTF16; - - ts_document_set_input(document, spy_input->input()); - ts_document_invalidate(document); - ts_document_parse(document); - }); - - it("measures columns in bytes", [&]() { - const char16_t content[] = u"[true, false]"; - spy_input->content = string((const char *)content, sizeof(content)); - spy_input->encoding = TSInputEncodingUTF16; - TSInput input = spy_input->input(); - - ts_document_set_input(document, input); - ts_document_invalidate(document); - ts_document_parse(document); - root = ts_document_root_node(document); - AssertThat(ts_node_end_point(root), Equals({0, 28})); - }); - - it("allows the input to be retrieved later", [&]() { - ts_document_set_input(document, spy_input->input()); - AssertThat(ts_document_input(document).payload, Equals(spy_input)); - AssertThat(ts_document_input(document).read, Equals(spy_input->input().read)); - AssertThat(ts_document_input(document).seek, Equals(spy_input->input().seek)); - }); - - it("does not assume that the document's text has changed", [&]() { - ts_document_set_input(document, spy_input->input()); - AssertThat(ts_document_root_node(document), Equals(root)); - AssertThat(ts_node_has_changes(root), IsFalse()); - AssertThat(spy_input->strings_read(), IsEmpty()); - }); - - it("reads text from the new input for future parses", [&]() { - ts_document_set_input(document, spy_input->input()); - - // Insert 'null', delete '1'. - TSInputEdit edit = {}; - edit.start_point.column = edit.start_byte = strlen("{\"key\": ["); - edit.extent_added.column = edit.bytes_added = 4; - edit.extent_removed.column = edit.bytes_removed = 1; - - ts_document_edit(document, edit); - ts_document_parse(document); - - TSNode new_root = ts_document_root_node(document); - assert_node_string_equals( - new_root, - "(value (object (pair (string) (array (null) (number)))))"); - AssertThat(spy_input->strings_read(), Equals(vector({" [null, 2" }))); - }); - - it("allows setting input string with length", [&]() { - const char content[] = { '1' }; - ts_document_set_input_string_with_length(document, content, 1); - ts_document_parse(document); - TSNode new_root = ts_document_root_node(document); - AssertThat(ts_node_end_byte(new_root), Equals(1)); - assert_node_string_equals( - new_root, - "(value (number))"); - }); - - it("reads from the new input correctly when the old input was blank", [&]() { - ts_document_set_input_string(document, ""); - ts_document_parse(document); - TSNode new_root = ts_document_root_node(document); - AssertThat(ts_node_end_byte(new_root), Equals(0)); - assert_node_string_equals( - new_root, - "(ERROR)"); - - ts_document_set_input_string(document, "1"); - ts_document_parse(document); - new_root = ts_document_root_node(document); - AssertThat(ts_node_end_byte(new_root), Equals(1)); - assert_node_string_equals( - new_root, - "(value (number))"); - }); - }); - - describe("set_language(language)", [&]() { - before_each([&]() { - ts_document_set_input_string(document, "{\"key\": [1, 2]}\n"); - }); - - it("uses the given language for future parses", [&]() { - ts_document_set_language(document, load_real_language("json")); - ts_document_parse(document); - - root = ts_document_root_node(document); - assert_node_string_equals( - root, - "(value (object (pair (string) (array (number) (number)))))"); - }); - - it("clears out any previous tree", [&]() { - ts_document_set_language(document, load_real_language("json")); - ts_document_parse(document); - - ts_document_set_language(document, load_real_language("javascript")); - AssertThat(ts_document_root_node(document).subtree, Equals(nullptr)); - - ts_document_parse(document); - root = ts_document_root_node(document); - assert_node_string_equals( - root, - "(program (expression_statement " - "(object (pair (string) (array (number) (number))))))"); - }); - - it("does not allow setting a language with a different version number", [&]() { - TSLanguage language = *load_real_language("json"); - AssertThat(ts_language_version(&language), Equals(TREE_SITTER_LANGUAGE_VERSION)); - - language.version++; - AssertThat(ts_language_version(&language), !Equals(TREE_SITTER_LANGUAGE_VERSION)); - - ts_document_set_language(document, &language); - AssertThat(ts_document_language(document), Equals(nullptr)); - }); - }); - - describe("set_logger(TSLogger)", [&]() { - SpyLogger *logger; - - before_each([&]() { - logger = new SpyLogger(); - ts_document_set_language(document, load_real_language("json")); - ts_document_set_input_string(document, "[1, 2]"); - }); - - after_each([&]() { - delete logger; - }); - - it("calls the debugger with a message for each parse action", [&]() { - ts_document_set_logger(document, logger->logger()); - ts_document_parse(document); - - AssertThat(logger->messages, Contains("new_parse")); - AssertThat(logger->messages, Contains("skip character:' '")); - AssertThat(logger->messages, Contains("consume character:'['")); - AssertThat(logger->messages, Contains("consume character:'1'")); - AssertThat(logger->messages, Contains("reduce sym:array, child_count:4")); - AssertThat(logger->messages, Contains("accept")); - }); - - it("allows the debugger to be retrieved later", [&]() { - ts_document_set_logger(document, logger->logger()); - AssertThat(ts_document_logger(document).payload, Equals(logger)); - }); - - describe("disabling debugging", [&]() { - before_each([&]() { - ts_document_set_logger(document, logger->logger()); - ts_document_set_logger(document, {NULL, NULL}); - }); - - it("does not call the debugger any more", [&]() { - ts_document_parse(document); - AssertThat(logger->messages, IsEmpty()); - }); - }); - }); - - describe("parse_and_get_changed_ranges()", [&]() { - SpyInput *input; - - before_each([&]() { - ts_document_set_language(document, load_real_language("javascript")); - input = new SpyInput("{a: null};\n", 3); - ts_document_set_input(document, input->input()); - ts_document_parse(document); - assert_node_string_equals( - ts_document_root_node(document), - "(program (expression_statement (object (pair (property_identifier) (null)))))"); - }); - - after_each([&]() { - delete input; - }); - - auto get_invalidated_ranges_for_edit = [&](std::function callback) -> vector { - TSInputEdit edit = callback(); - ts_document_edit(document, edit); - - TSRange *ranges; - uint32_t range_count = 0; - ts_document_parse_and_get_changed_ranges(document, &ranges, &range_count); - - vector result; - for (size_t i = 0; i < range_count; i++) { - result.push_back(ranges[i]); - } - ts_free(ranges); - return result; - }; - - it("reports changes when one token has been updated", [&]() { - // Replace `null` with `nothing` - auto ranges = get_invalidated_ranges_for_edit([&]() { - return input->replace(input->content.find("ull"), 1, "othing"); - }); - - AssertThat(ranges, Equals(vector({ - TSRange{ - point(0, input->content.find("nothing")), - point(0, input->content.find("}")) - }, - }))); - - // Replace `nothing` with `null` again - ranges = get_invalidated_ranges_for_edit([&]() { - return input->undo(); - }); - - AssertThat(ranges, Equals(vector({ - TSRange{ - point(0, input->content.find("null")), - point(0, input->content.find("}")) - }, - }))); - }); - - it("reports no changes when leading whitespace has changed (regression)", [&]() { - input->chars_per_chunk = 80; - - // Insert leading whitespace - auto ranges = get_invalidated_ranges_for_edit([&]() { - return input->replace(0, 0, "\n"); - }); - assert_node_string_equals( - ts_document_root_node(document), - "(program (expression_statement (object (pair (property_identifier) (null)))))"); - AssertThat(ranges, Equals(vector({}))); - - // Remove leading whitespace - ranges = get_invalidated_ranges_for_edit([&]() { - return input->undo(); - }); - assert_node_string_equals( - ts_document_root_node(document), - "(program (expression_statement (object (pair (property_identifier) (null)))))"); - AssertThat(ranges, Equals(vector({}))); - - // Insert leading whitespace again - ranges = get_invalidated_ranges_for_edit([&]() { - return input->replace(0, 0, "\n"); - }); - assert_node_string_equals( - ts_document_root_node(document), - "(program (expression_statement (object (pair (property_identifier) (null)))))"); - AssertThat(ranges, Equals(vector({}))); - }); - - it("reports changes when tokens have been appended", [&]() { - // Add a second key-value pair - auto ranges = get_invalidated_ranges_for_edit([&]() { - return input->replace(input->content.find("}"), 0, ", b: false"); - }); - - AssertThat(ranges, Equals(vector({ - TSRange{ - point(0, input->content.find(",")), - point(0, input->content.find("}")) - }, - }))); - - // Add a third key-value pair in between the first two - ranges = get_invalidated_ranges_for_edit([&]() { - return input->replace(input->content.find(", b"), 0, ", c: 1"); - }); - - assert_node_string_equals( - ts_document_root_node(document), - "(program (expression_statement (object " - "(pair (property_identifier) (null)) " - "(pair (property_identifier) (number)) " - "(pair (property_identifier) (false)))))"); - - AssertThat(ranges, Equals(vector({ - TSRange{ - point(0, input->content.find(", c")), - point(0, input->content.find(", b")) - }, - }))); - - // Delete the middle pair. - ranges = get_invalidated_ranges_for_edit([&]() { - return input->undo(); - }); - - assert_node_string_equals( - ts_document_root_node(document), - "(program (expression_statement (object " - "(pair (property_identifier) (null)) " - "(pair (property_identifier) (false)))))"); - - AssertThat(ranges, IsEmpty()); - - // Delete the second pair. - ranges = get_invalidated_ranges_for_edit([&]() { - return input->undo(); - }); - - assert_node_string_equals( - ts_document_root_node(document), - "(program (expression_statement (object " - "(pair (property_identifier) (null)))))"); - - AssertThat(ranges, IsEmpty()); - }); - - it("reports changes when trees have been wrapped", [&]() { - // Wrap the object in an assignment expression. - auto ranges = get_invalidated_ranges_for_edit([&]() { - return input->replace(input->content.find("null"), 0, "b === "); - }); - - assert_node_string_equals( - ts_document_root_node(document), - "(program (expression_statement (object " - "(pair (property_identifier) (binary_expression (identifier) (null))))))"); - - AssertThat(ranges, Equals(vector({ - TSRange{ - point(0, input->content.find("b ===")), - point(0, input->content.find("}")) - }, - }))); - }); - }); - - describe("parse_with_options(options)", [&]() { - it("halts as soon as an error is found if the halt_on_error flag is set", [&]() { - string input_string = "[1, null, error, 3]"; - ts_document_set_language(document, load_real_language("json")); - ts_document_set_input_string(document, input_string.c_str()); - - TSParseOptions options = {}; - options.changed_ranges = nullptr; - - options.halt_on_error = false; - ts_document_parse_with_options(document, options); - root = ts_document_root_node(document); - assert_node_string_equals( - root, - "(value (array (number) (null) (ERROR (UNEXPECTED 'e')) (number)))"); - - ts_document_invalidate(document); - - options.halt_on_error = true; - ts_document_parse_with_options(document, options); - root = ts_document_root_node(document); - assert_node_string_equals( - root, - "(ERROR (number) (null))"); - - AssertThat(ts_node_end_byte(root), Equals(input_string.size())); - }); - - it("does not insert missing tokens if the halt_on_error flag is set", [&]() { - string input_string = "[1, null, 3"; - ts_document_set_language(document, load_real_language("json")); - ts_document_set_input_string(document, input_string.c_str()); - - TSParseOptions options = {}; - options.changed_ranges = nullptr; - - options.halt_on_error = false; - ts_document_parse_with_options(document, options); - root = ts_document_root_node(document); - assert_node_string_equals( - root, - "(value (array (number) (null) (number) (MISSING)))"); - - ts_document_invalidate(document); - - options.halt_on_error = true; - ts_document_parse_with_options(document, options); - root = ts_document_root_node(document); - assert_node_string_equals( - root, - "(ERROR (number) (null) (number))"); - - AssertThat(ts_node_end_byte(root), Equals(input_string.size())); - }); - - it("can parse valid code with the halt_on_error flag set", [&]() { - string input_string = "[1, null, 3]"; - ts_document_set_language(document, load_real_language("json")); - ts_document_set_input_string(document, input_string.c_str()); - - TSParseOptions options = {}; - options.changed_ranges = nullptr; - options.halt_on_error = true; - ts_document_parse_with_options(document, options); - root = ts_document_root_node(document); - assert_node_string_equals( - root, - "(value (array (number) (null) (number)))"); - }); - }); -}); - -END_TEST diff --git a/test/runtime/language_test.cc b/test/runtime/language_test.cc index 4726c4ba..747327c0 100644 --- a/test/runtime/language_test.cc +++ b/test/runtime/language_test.cc @@ -28,13 +28,12 @@ describe("Language", []() { } })JSON"); - TSDocument *document = ts_document_new(); + TSParser *parser = ts_parser_new(); const TSLanguage *language = load_test_language("aliased_rules", compile_result); - ts_document_set_language(document, language); - ts_document_set_input_string(document, "b"); - ts_document_parse(document); + ts_parser_set_language(parser, language); + TSTree *tree = ts_parser_parse_string(parser, nullptr, "b", 1); - TSNode root_node = ts_document_root_node(document); + TSNode root_node = ts_tree_root_node(tree); char *string = ts_node_string(root_node); AssertThat(string, Equals("(a (c))")); @@ -47,7 +46,8 @@ describe("Language", []() { AssertThat(ts_language_symbol_type(language, aliased_symbol), Equals(TSSymbolTypeRegular)); ts_free(string); - ts_document_free(document); + ts_parser_delete(parser); + ts_tree_delete(tree); }); }); }); diff --git a/test/runtime/node_test.cc b/test/runtime/node_test.cc index 808084be..4c306897 100644 --- a/test/runtime/node_test.cc +++ b/test/runtime/node_test.cc @@ -62,21 +62,22 @@ string grammar_with_aliases_and_extras = R"JSON({ })JSON"; describe("Node", [&]() { - TSDocument *document; + TSParser *parser; + TSTree *tree; TSNode root_node; before_each([&]() { record_alloc::start(); - document = ts_document_new(); - ts_document_set_language(document, load_real_language("json")); - ts_document_set_input_string(document, json_string.c_str()); - ts_document_parse(document); - root_node = ts_node_child(ts_document_root_node(document), 0); + parser = ts_parser_new(); + ts_parser_set_language(parser, load_real_language("json")); + tree = ts_parser_parse_string(parser, nullptr, json_string.c_str(), json_string.size()); + root_node = ts_node_child(ts_tree_root_node(tree), 0); }); after_each([&]() { - ts_document_free(document); + ts_parser_delete(parser); + ts_tree_delete(tree); record_alloc::stop(); AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); @@ -157,16 +158,17 @@ describe("Node", [&]() { AssertThat(ts_node_parent(number_node), Equals(root_node)); AssertThat(ts_node_parent(false_node), Equals(root_node)); AssertThat(ts_node_parent(object_node), Equals(root_node)); - AssertThat(ts_node_parent(ts_document_root_node(document)).subtree, Equals(nullptr)); + AssertThat(ts_node_parent(ts_tree_root_node(tree)).subtree, Equals(nullptr)); }); it("works correctly when the node contains aliased children and extras", [&]() { TSCompileResult compile_result = ts_compile_grammar(grammar_with_aliases_and_extras.c_str()); const TSLanguage *language = load_test_language("aliases_and_extras", compile_result); - ts_document_set_language(document, language); - ts_document_set_input_string(document, "b ... b ... b"); - ts_document_parse(document); - root_node = ts_document_root_node(document); + ts_parser_set_language(parser, language); + + ts_tree_delete(tree); + tree = ts_parser_parse_string(parser, nullptr, "b ... b ... b", 13); + root_node = ts_tree_root_node(tree); char *node_string = ts_node_string(root_node); AssertThat(node_string, Equals("(a (b) (comment) (B) (comment) (b))")); @@ -179,7 +181,10 @@ describe("Node", [&]() { AssertThat(ts_node_type(ts_node_named_child(root_node, 3)), Equals("comment")); AssertThat(ts_node_type(ts_node_named_child(root_node, 4)), Equals("b")); - AssertThat(ts_node_symbol(ts_node_named_child(root_node, 0)), !Equals(ts_node_symbol(ts_node_named_child(root_node, 2)))); + AssertThat( + ts_node_symbol(ts_node_named_child(root_node, 0)), + !Equals(ts_node_symbol(ts_node_named_child(root_node, 2))) + ); }); }); @@ -323,7 +328,7 @@ describe("Node", [&]() { AssertThat(ts_node_parent(child5), Equals(root_node)); AssertThat(ts_node_parent(child6), Equals(root_node)); AssertThat(ts_node_parent(child7), Equals(root_node)); - AssertThat(ts_node_parent(ts_document_root_node(document)).subtree, Equals(nullptr)); + AssertThat(ts_node_parent(ts_tree_root_node(tree)).subtree, Equals(nullptr)); }); }); @@ -483,9 +488,10 @@ describe("Node", [&]() { it("works in the presence of multi-byte characters", [&]() { string input_string = "[\"αβγδ\", \"αβγδ\"]"; - ts_document_set_input_string(document, input_string.c_str()); - ts_document_parse(document); - TSNode root_node = ts_document_root_node(document); + + ts_tree_delete(tree); + tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size()); + TSNode root_node = ts_tree_root_node(tree); uint32_t comma_position = input_string.find(","); TSNode node1 = ts_node_descendant_for_byte_range(root_node, comma_position, comma_position); @@ -518,23 +524,23 @@ describe("Node", [&]() { }); describe("TreeCursor", [&]() { - TSDocument *document; + TSParser *parser; + TSTree *tree; TSTreeCursor *cursor; before_each([&]() { record_alloc::start(); - document = ts_document_new(); - ts_document_set_language(document, load_real_language("json")); - ts_document_set_input_string(document, json_string.c_str()); - ts_document_parse(document); - - cursor = ts_document_tree_cursor(document); + parser = ts_parser_new(); + ts_parser_set_language(parser, load_real_language("json")); + tree = ts_parser_parse_string(parser, nullptr, json_string.c_str(), json_string.size()); + cursor = ts_tree_cursor_new(tree); }); after_each([&]() { + ts_tree_delete(tree); ts_tree_cursor_delete(cursor); - ts_document_free(document); + ts_parser_delete(parser); record_alloc::stop(); AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); diff --git a/test/runtime/parser_test.cc b/test/runtime/parser_test.cc index 2cf70b69..e9cfbe72 100644 --- a/test/runtime/parser_test.cc +++ b/test/runtime/parser_test.cc @@ -1,17 +1,20 @@ #include "test_helper.h" #include "runtime/alloc.h" +#include "runtime/language.h" #include "helpers/record_alloc.h" #include "helpers/spy_input.h" #include "helpers/load_language.h" #include "helpers/record_alloc.h" #include "helpers/point_helpers.h" +#include "helpers/spy_logger.h" #include "helpers/stderr_logger.h" #include "helpers/dedent.h" START_TEST describe("Parser", [&]() { - TSDocument *document; + TSParser *parser; + TSTree *tree; SpyInput *input; TSNode root; size_t chunk_size; @@ -21,14 +24,16 @@ describe("Parser", [&]() { chunk_size = 3; input = nullptr; - document = ts_document_new(); + tree = nullptr; + parser = ts_parser_new(); if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) { - ts_document_print_debugging_graphs(document, true); + ts_parser_print_debugging_graphs(parser, true); } }); after_each([&]() { - if (document) ts_document_free(document); + if (parser) ts_parser_delete(parser); + if (tree) ts_tree_delete(tree); if (input) delete input; record_alloc::stop(); @@ -37,10 +42,8 @@ describe("Parser", [&]() { auto set_text = [&](string text) { input = new SpyInput(text, chunk_size); - ts_document_set_input(document, input->input()); - ts_document_parse(document); - - root = ts_document_root_node(document); + tree = ts_parser_parse(parser, nullptr, input->input()); + root = ts_tree_root_node(tree); AssertThat(ts_node_end_byte(root), Equals(text.size())); input->clear(); }; @@ -48,10 +51,13 @@ describe("Parser", [&]() { auto replace_text = [&](size_t position, size_t length, string new_text) { size_t prev_size = ts_node_end_byte(root); - ts_document_edit(document, input->replace(position, length, new_text)); - ts_document_parse(document); + TSInputEdit edit = input->replace(position, length, new_text); + ts_tree_edit(tree, &edit); + TSTree *new_tree = ts_parser_parse(parser, tree, input->input()); + ts_tree_delete(tree); + tree = new_tree; - root = ts_document_root_node(document); + root = ts_tree_root_node(tree); size_t new_size = ts_node_end_byte(root); AssertThat(new_size, Equals(prev_size - length + new_text.size())); }; @@ -65,12 +71,15 @@ describe("Parser", [&]() { }; auto undo = [&]() { - ts_document_edit(document, input->undo()); - ts_document_parse(document); + TSInputEdit edit = input->undo(); + ts_tree_edit(tree, &edit); + TSTree *new_tree = ts_parser_parse(parser, tree, input->input()); + ts_tree_delete(tree); + tree = new_tree; }; auto assert_root_node = [&](const string &expected) { - TSNode node = ts_document_root_node(document); + TSNode node = ts_tree_root_node(tree); char *node_string = ts_node_string(node); string actual(node_string); ts_free(node_string); @@ -86,11 +95,9 @@ describe("Parser", [&]() { describe("handling errors", [&]() { describe("when there is an invalid substring right before a valid token", [&]() { it("computes the error node's size and position correctly", [&]() { - ts_document_set_language(document, load_real_language("json")); + ts_parser_set_language(parser, load_real_language("json")); set_text(" [123, @@@@@, true]"); - - assert_root_node( - "(value (array (number) (ERROR (UNEXPECTED '@')) (true)))"); + assert_root_node("(value (array (number) (ERROR (UNEXPECTED '@')) (true)))"); TSNode error = ts_node_named_child(ts_node_child(root, 0), 1); AssertThat(ts_node_type(error), Equals("ERROR")); @@ -111,7 +118,7 @@ describe("Parser", [&]() { describe("when there is an unexpected string in the middle of a token", [&]() { it("computes the error node's size and position correctly", [&]() { - ts_document_set_language(document, load_real_language("json")); + ts_parser_set_language(parser, load_real_language("json")); set_text(" [123, faaaaalse, true]"); assert_root_node( @@ -138,11 +145,10 @@ describe("Parser", [&]() { describe("when there is one unexpected token between two valid tokens", [&]() { it("computes the error node's size and position correctly", [&]() { - ts_document_set_language(document, load_real_language("json")); + ts_parser_set_language(parser, load_real_language("json")); set_text(" [123, true false, true]"); - assert_root_node( - "(value (array (number) (true) (ERROR (false)) (true)))"); + assert_root_node("(value (array (number) (true) (ERROR (false)) (true)))"); TSNode error = ts_node_named_child(ts_node_child(root, 0), 2); AssertThat(ts_node_type(error), Equals("ERROR")); @@ -157,26 +163,23 @@ describe("Parser", [&]() { describe("when there is an unexpected string at the end of a token", [&]() { it("computes the error's size and position correctly", [&]() { - ts_document_set_language(document, load_real_language("json")); + ts_parser_set_language(parser, load_real_language("json")); set_text(" [123, \"hi\n, true]"); - - assert_root_node( - "(value (array (number) (ERROR (UNEXPECTED '\\n')) (true)))"); + assert_root_node("(value (array (number) (ERROR (UNEXPECTED '\\n')) (true)))"); }); }); describe("when there is an unterminated error", [&]() { it("maintains a consistent tree", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("a; ' this string never ends"); - assert_root_node( - "(program (expression_statement (identifier)) (ERROR (UNEXPECTED EOF)))"); + assert_root_node("(program (expression_statement (identifier)) (ERROR (UNEXPECTED EOF)))"); }); }); describe("when there are extra tokens at the end of the viable prefix", [&]() { it("does not include them in the error node", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text( "var x;\n" "\n" @@ -196,20 +199,64 @@ describe("Parser", [&]() { char *string = (char *)malloc(1); string[0] = '\xdf'; - ts_document_set_language(document, load_real_language("json")); - ts_document_set_input_string_with_length(document, string, 1); - ts_document_parse(document); + ts_parser_set_language(parser, load_real_language("json")); + tree = ts_parser_parse_string(parser, nullptr, string, 1); free(string); - assert_root_node("(ERROR (UNEXPECTED INVALID))"); }); + + describe("when halt_on_error is set to true", [&]() { + it("halts as soon as an error is found if the halt_on_error flag is set", [&]() { + string input_string = "[1, null, error, 3]"; + ts_parser_set_language(parser, load_real_language("json")); + + tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size()); + root = ts_tree_root_node(tree); + assert_root_node("(value (array (number) (null) (ERROR (UNEXPECTED 'e')) (number)))"); + + ts_parser_halt_on_error(parser, true); + + ts_tree_delete(tree); + tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size()); + root = ts_tree_root_node(tree); + assert_root_node("(ERROR (number) (null))"); + AssertThat(ts_node_end_byte(root), Equals(input_string.size())); + }); + + it("does not insert missing tokens if the halt_on_error flag is set", [&]() { + string input_string = "[1, null, 3"; + ts_parser_set_language(parser, load_real_language("json")); + + tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size()); + root = ts_tree_root_node(tree); + assert_root_node("(value (array (number) (null) (number) (MISSING)))"); + + ts_parser_halt_on_error(parser, true); + + ts_tree_delete(tree); + tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size()); + root = ts_tree_root_node(tree); + assert_root_node("(ERROR (number) (null) (number))"); + AssertThat(ts_node_end_byte(root), Equals(input_string.size())); + }); + + it("can parse valid code with the halt_on_error flag set", [&]() { + string input_string = "[1, null, 3]"; + ts_parser_set_language(parser, load_real_language("json")); + + ts_parser_halt_on_error(parser, true); + tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size()); + root = ts_tree_root_node(tree); + assert_root_node("(value (array (number) (null) (number)))"); + }); + }); }); describe("editing", [&]() { describe("creating new tokens near the end of the input", [&]() { it("updates the parse tree and re-reads only the changed portion of the text", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("x * (100 + abc);"); assert_root_node( @@ -242,7 +289,7 @@ describe("Parser", [&]() { it("updates the parse tree and re-reads only the changed portion of the input", [&]() { chunk_size = 2; - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("123 + 456 * (10 + x);"); assert_root_node( @@ -268,7 +315,7 @@ describe("Parser", [&]() { describe("introducing an error", [&]() { it("gives the error the right size", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("var x = y;"); assert_root_node( @@ -291,7 +338,7 @@ describe("Parser", [&]() { describe("into the middle of an existing token", [&]() { it("updates the parse tree", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("abc * 123;"); assert_root_node( @@ -310,7 +357,7 @@ describe("Parser", [&]() { describe("at the end of an existing token", [&]() { it("updates the parse tree", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("abc * 123;"); assert_root_node( @@ -329,7 +376,7 @@ describe("Parser", [&]() { describe("inserting text into a node containing a extra token", [&]() { it("updates the parse tree", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("123 *\n" "// a-comment\n" "abc;"); @@ -356,7 +403,7 @@ describe("Parser", [&]() { describe("when a critical token is removed", [&]() { it("updates the parse tree, creating an error", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("123 * 456; 789 * 123;"); assert_root_node( @@ -376,7 +423,7 @@ describe("Parser", [&]() { describe("with external tokens", [&]() { it("maintains the external scanner's state during incremental parsing", [&]() { - ts_document_set_language(document, load_real_language("python")); + ts_parser_set_language(parser, load_real_language("python")); string text = dedent(R"PYTHON( if a: print b @@ -404,7 +451,7 @@ describe("Parser", [&]() { }); it("does not try to reuse nodes that are within the edited region", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("{ x: (b.c) };"); assert_root_node( @@ -417,23 +464,12 @@ describe("Parser", [&]() { "(program (expression_statement (object (pair " "(property_identifier) (member_expression (identifier) (property_identifier))))))"); }); - - it("updates the document's parse count", [&]() { - ts_document_set_language(document, load_real_language("javascript")); - AssertThat(ts_document_parse_count(document), Equals(0)); - - set_text("{ x: (b.c) };"); - AssertThat(ts_document_parse_count(document), Equals(1)); - - insert_text(strlen("{ x"), "yz"); - AssertThat(ts_document_parse_count(document), Equals(2)); - }); }); describe("lexing", [&]() { describe("handling tokens containing wildcard patterns (e.g. comments)", [&]() { - it("terminates them at the end of the document", [&]() { - ts_document_set_language(document, load_real_language("javascript")); + it("terminates them at the end of the string", [&]() { + ts_parser_set_language(parser, load_real_language("javascript")); set_text("x; // this is a comment"); assert_root_node( @@ -448,7 +484,7 @@ describe("Parser", [&]() { it("recognizes UTF8 characters as single characters", [&]() { // 'ΩΩΩ — ΔΔ'; - ts_document_set_language(document, load_real_language("javascript")); + ts_parser_set_language(parser, load_real_language("javascript")); set_text("'\u03A9\u03A9\u03A9 \u2014 \u0394\u0394';"); assert_root_node( @@ -460,14 +496,120 @@ describe("Parser", [&]() { it("handles non-UTF8 characters", [&]() { const char *string = "cons\xeb\x00e=ls\x83l6hi');\x0a"; - ts_document_set_language(document, load_real_language("javascript")); - ts_document_set_input_string(document, string); - ts_document_parse(document); - - TSNode root = ts_document_root_node(document); + ts_parser_set_language(parser, load_real_language("javascript")); + tree = ts_parser_parse_string(parser, nullptr, string, strlen(string)); + TSNode root = ts_tree_root_node(tree); AssertThat(ts_node_end_byte(root), Equals(strlen(string))); }); }); + + describe("handling TSInputs", [&]() { + SpyInput *spy_input; + + before_each([&]() { + spy_input = new SpyInput("{\"key\": [null, 2]}", 3); + ts_parser_set_language(parser, load_real_language("json")); + }); + + after_each([&]() { + delete spy_input; + }); + + it("handles UTF16 encodings", [&]() { + const char16_t content[] = u"[true, false]"; + spy_input->content = string((const char *)content, sizeof(content)); + spy_input->encoding = TSInputEncodingUTF16; + + tree = ts_parser_parse(parser, nullptr, spy_input->input()); + root = ts_tree_root_node(tree); + assert_root_node( + "(value (array (true) (false)))"); + }); + + it("handles truncated UTF16 data", [&]() { + const char content[1] = { '\0' }; + spy_input->content = string(content, sizeof(content)); + spy_input->encoding = TSInputEncodingUTF16; + + tree = ts_parser_parse(parser, nullptr, spy_input->input()); + }); + + it("measures columns in bytes", [&]() { + const char16_t content[] = u"[true, false]"; + spy_input->content = string((const char *)content, sizeof(content)); + spy_input->encoding = TSInputEncodingUTF16; + + tree = ts_parser_parse(parser, nullptr, spy_input->input()); + root = ts_tree_root_node(tree); + AssertThat(ts_node_end_point(root), Equals({0, 28})); + }); + }); + + describe("set_language(language)", [&]() { + string input_string = "{\"key\": [1, 2]}\n"; + + it("uses the given language for future parses", [&]() { + ts_parser_set_language(parser, load_real_language("json")); + tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size()); + + root = ts_tree_root_node(tree); + assert_root_node( + "(value (object (pair (string) (array (number) (number)))))"); + }); + + it("does not allow setting a language with a different version number", [&]() { + TSLanguage language = *load_real_language("json"); + AssertThat(ts_language_version(&language), Equals(TREE_SITTER_LANGUAGE_VERSION)); + + language.version++; + AssertThat(ts_language_version(&language), !Equals(TREE_SITTER_LANGUAGE_VERSION)); + + AssertThat(ts_parser_set_language(parser, &language), IsFalse()); + AssertThat(ts_parser_language(parser), Equals(nullptr)); + }); + }); + + describe("set_logger(TSLogger)", [&]() { + SpyLogger *logger; + + before_each([&]() { + logger = new SpyLogger(); + ts_parser_set_language(parser, load_real_language("json")); + }); + + after_each([&]() { + delete logger; + }); + + it("calls the debugger with a message for each parse action", [&]() { + ts_parser_set_logger(parser, logger->logger()); + tree = ts_parser_parse_string(parser, nullptr, "[ 1, 2, 3 ]", 11); + + AssertThat(logger->messages, Contains("new_parse")); + AssertThat(logger->messages, Contains("skip character:' '")); + AssertThat(logger->messages, Contains("consume character:'['")); + AssertThat(logger->messages, Contains("consume character:'1'")); + AssertThat(logger->messages, Contains("reduce sym:array, child_count:4")); + AssertThat(logger->messages, Contains("accept")); + }); + + it("allows the debugger to be retrieved later", [&]() { + ts_parser_set_logger(parser, logger->logger()); + AssertThat(ts_parser_logger(parser).payload, Equals(logger)); + }); + + describe("disabling debugging", [&]() { + before_each([&]() { + ts_parser_set_logger(parser, logger->logger()); + ts_parser_set_logger(parser, {NULL, NULL}); + }); + + it("does not call the debugger any more", [&]() { + tree = ts_parser_parse_string(parser, nullptr, "{}", 2); + AssertThat(logger->messages, IsEmpty()); + }); + }); + }); }); END_TEST diff --git a/test/runtime/stack_test.cc b/test/runtime/stack_test.cc index 99a979d1..ae5b1589 100644 --- a/test/runtime/stack_test.cc +++ b/test/runtime/stack_test.cc @@ -76,7 +76,7 @@ describe("Stack", [&]() { before_each([&]() { record_alloc::start(); - ts_subtree_pool_init(&pool); + pool = ts_subtree_pool_new(10); stack = ts_stack_new(&pool); TSLanguage dummy_language; diff --git a/test/runtime/subtree_test.cc b/test/runtime/subtree_test.cc index d56be01d..093e68db 100644 --- a/test/runtime/subtree_test.cc +++ b/test/runtime/subtree_test.cc @@ -41,7 +41,7 @@ describe("Subtree", []() { SubtreePool pool; before_each([&]() { - ts_subtree_pool_init(&pool); + pool = ts_subtree_pool_new(10); }); after_each([&]() { diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc new file mode 100644 index 00000000..a87c754b --- /dev/null +++ b/test/runtime/tree_test.cc @@ -0,0 +1,200 @@ +#include "test_helper.h" +#include "runtime/alloc.h" +#include "helpers/record_alloc.h" +#include "helpers/stream_methods.h" +#include "helpers/tree_helpers.h" +#include "helpers/point_helpers.h" +#include "helpers/spy_logger.h" +#include "helpers/stderr_logger.h" +#include "helpers/spy_input.h" +#include "helpers/load_language.h" + +TSPoint point(uint32_t row, uint32_t column) { + TSPoint result = {row, column}; + return result; +} + +START_TEST + +describe("Tree", [&]() { + TSParser *parser; + SpyInput *input; + TSTree *tree; + + before_each([&]() { + parser = ts_parser_new(); + }); + + after_each([&]() { + ts_parser_delete(parser); + }); + + auto assert_root_node = [&](const string &expected) { + TSNode node = ts_tree_root_node(tree); + char *node_string = ts_node_string(node); + string actual(node_string); + ts_free(node_string); + AssertThat(actual, Equals(expected)); + }; + + describe("get_changed_ranges()", [&]() { + before_each([&]() { + ts_parser_set_language(parser, load_real_language("javascript")); + input = new SpyInput("{a: null};\n", 3); + tree = ts_parser_parse(parser, nullptr, input->input()); + + assert_root_node( + "(program (expression_statement (object (pair (property_identifier) (null)))))" + ); + }); + + after_each([&]() { + ts_tree_delete(tree); + delete input; + }); + + auto get_changed_ranges_for_edit = [&](function fn) -> vector { + TSInputEdit edit = fn(); + ts_tree_edit(tree, &edit); + + uint32_t range_count = 0; + TSTree *new_tree = ts_parser_parse(parser, tree, input->input()); + TSRange *ranges = ts_tree_get_changed_ranges(tree, new_tree, &range_count); + ts_tree_delete(tree); + tree = new_tree; + + vector result; + for (size_t i = 0; i < range_count; i++) { + result.push_back(ranges[i]); + } + + ts_free(ranges); + return result; + }; + + it("reports changes when one token has been updated", [&]() { + // Replace `null` with `nothing` + auto ranges = get_changed_ranges_for_edit([&]() { + return input->replace(input->content.find("ull"), 1, "othing"); + }); + AssertThat(ranges, Equals(vector({ + TSRange{ + point(0, input->content.find("nothing")), + point(0, input->content.find("}")) + }, + }))); + + // Replace `nothing` with `null` again + ranges = get_changed_ranges_for_edit([&]() { + return input->undo(); + }); + AssertThat(ranges, Equals(vector({ + TSRange{ + point(0, input->content.find("null")), + point(0, input->content.find("}")) + }, + }))); + }); + + it("reports no changes when leading whitespace has changed (regression)", [&]() { + input->chars_per_chunk = 80; + + // Insert leading whitespace + auto ranges = get_changed_ranges_for_edit([&]() { + return input->replace(0, 0, "\n"); + }); + assert_root_node( + "(program (expression_statement (object (pair (property_identifier) (null)))))" + ); + AssertThat(ranges, IsEmpty()); + + // Remove leading whitespace + ranges = get_changed_ranges_for_edit([&]() { + return input->undo(); + }); + assert_root_node( + "(program (expression_statement (object (pair (property_identifier) (null)))))" + ); + AssertThat(ranges, IsEmpty()); + + // Insert leading whitespace again + ranges = get_changed_ranges_for_edit([&]() { + return input->replace(0, 0, "\n"); + }); + assert_root_node( + "(program (expression_statement (object (pair (property_identifier) (null)))))" + ); + AssertThat(ranges, IsEmpty()); + }); + + it("reports changes when tokens have been appended", [&]() { + // Add a second key-value pair + auto ranges = get_changed_ranges_for_edit([&]() { + return input->replace(input->content.find("}"), 0, ", b: false"); + }); + AssertThat(ranges, Equals(vector({ + TSRange{ + point(0, input->content.find(",")), + point(0, input->content.find("}")) + }, + }))); + + // Add a third key-value pair in between the first two + ranges = get_changed_ranges_for_edit([&]() { + return input->replace(input->content.find(", b"), 0, ", c: 1"); + }); + assert_root_node( + "(program (expression_statement (object " + "(pair (property_identifier) (null)) " + "(pair (property_identifier) (number)) " + "(pair (property_identifier) (false)))))" + ); + AssertThat(ranges, Equals(vector({ + TSRange{ + point(0, input->content.find(", c")), + point(0, input->content.find(", b")) + }, + }))); + + // Delete the middle pair. + ranges = get_changed_ranges_for_edit([&]() { + return input->undo(); + }); + assert_root_node( + "(program (expression_statement (object " + "(pair (property_identifier) (null)) " + "(pair (property_identifier) (false)))))" + ); + AssertThat(ranges, IsEmpty()); + + // Delete the second pair. + ranges = get_changed_ranges_for_edit([&]() { + return input->undo(); + }); + assert_root_node( + "(program (expression_statement (object " + "(pair (property_identifier) (null)))))" + ); + AssertThat(ranges, IsEmpty()); + }); + + it("reports changes when trees have been wrapped", [&]() { + // Wrap the object in an assignment expression. + auto ranges = get_changed_ranges_for_edit([&]() { + return input->replace(input->content.find("null"), 0, "b === "); + }); + assert_root_node( + "(program (expression_statement (object " + "(pair (property_identifier) (binary_expression (identifier) (null))))))" + ); + AssertThat(ranges, Equals(vector({ + TSRange{ + point(0, input->content.find("b ===")), + point(0, input->content.find("}")) + }, + }))); + }); + }); +}); + +END_TEST diff --git a/tests.gyp b/tests.gyp index b964a507..ab0de485 100644 --- a/tests.gyp +++ b/tests.gyp @@ -66,12 +66,12 @@ 'test/integration/fuzzing-examples.cc', 'test/integration/real_grammars.cc', 'test/integration/test_grammars.cc', - 'test/runtime/document_test.cc', 'test/runtime/language_test.cc', 'test/runtime/node_test.cc', 'test/runtime/parser_test.cc', 'test/runtime/stack_test.cc', 'test/runtime/subtree_test.cc', + 'test/runtime/tree_test.cc', 'test/tests.cc', ], 'cflags': [ From 199a94cc26640c3add7b23757a1de510d7acf058 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 12:43:04 -0700 Subject: [PATCH 21/31] Allow the parser to print dot graphs to any file --- include/tree_sitter/runtime.h | 12 +++++----- src/runtime/parser.c | 36 ++++++++++++++-------------- test/benchmarks.cc | 2 +- test/integration/fuzzing-examples.cc | 2 +- test/integration/real_grammars.cc | 2 +- test/integration/test_grammars.cc | 2 +- test/runtime/parser_test.cc | 2 +- 7 files changed, 29 insertions(+), 29 deletions(-) diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index fa6fd919..acd2c3b3 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -34,6 +34,11 @@ typedef struct { uint32_t column; } TSPoint; +typedef struct { + TSPoint start; + TSPoint end; +} TSRange; + typedef struct { void *payload; const char *(*read)(void *payload, uint32_t *bytes_read); @@ -60,11 +65,6 @@ typedef struct { TSPoint extent_added; } TSInputEdit; -typedef struct { - TSPoint start; - TSPoint end; -} TSRange; - typedef struct { const void *subtree; const TSTree *tree; @@ -79,7 +79,7 @@ const TSLanguage *ts_parser_language(const TSParser *); bool ts_parser_set_language(TSParser *, const TSLanguage *); TSLogger ts_parser_logger(const TSParser *); void ts_parser_set_logger(TSParser *, TSLogger); -void ts_parser_print_debugging_graphs(TSParser *, bool); +void ts_parser_print_dot_graphs(TSParser *, FILE *); void ts_parser_halt_on_error(TSParser *, bool); TSTree *ts_parser_parse(TSParser *, const TSTree *, TSInput); TSTree *ts_parser_parse_string(TSParser *, const TSTree *, const char *, uint32_t); diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 1c7da6b1..02e96857 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -17,21 +17,21 @@ #include "runtime/tree.h" #define LOG(...) \ - if (self->lexer.logger.log || self->print_debugging_graphs) { \ + if (self->lexer.logger.log || self->dot_graph_file) { \ snprintf(self->lexer.debug_buffer, TREE_SITTER_SERIALIZATION_BUFFER_SIZE, __VA_ARGS__); \ ts_parser__log(self); \ } -#define LOG_STACK() \ - if (self->print_debugging_graphs) { \ - ts_stack_print_dot_graph(self->stack, self->language, stderr); \ - fputs("\n\n", stderr); \ +#define LOG_STACK() \ + if (self->dot_graph_file) { \ + ts_stack_print_dot_graph(self->stack, self->language, self->dot_graph_file); \ + fputs("\n\n", self->dot_graph_file); \ } -#define LOG_TREE() \ - if (self->print_debugging_graphs) { \ - ts_subtree_print_dot_graph(self->finished_tree, self->language, stderr); \ - fputs("\n", stderr); \ +#define LOG_TREE() \ + if (self->dot_graph_file) { \ + ts_subtree_print_dot_graph(self->finished_tree, self->language, self->dot_graph_file); \ + fputs("\n", self->dot_graph_file); \ } #define SYM_NAME(symbol) ts_language_symbol_name(self->language, symbol) @@ -58,7 +58,7 @@ struct TSParser { ReusableNode reusable_node; void *external_scanner_payload; bool in_ambiguity; - bool print_debugging_graphs; + FILE *dot_graph_file; bool halt_on_error; unsigned accept_count; }; @@ -89,13 +89,13 @@ static void ts_parser__log(TSParser *self) { ); } - if (self->print_debugging_graphs) { - fprintf(stderr, "graph {\nlabel=\""); + if (self->dot_graph_file) { + fprintf(self->dot_graph_file, "graph {\nlabel=\""); for (char *c = &self->lexer.debug_buffer[0]; *c != 0; c++) { - if (*c == '"') fputc('\\', stderr); - fputc(*c, stderr); + if (*c == '"') fputc('\\', self->dot_graph_file); + fputc(*c, self->dot_graph_file); } - fprintf(stderr, "\"\n}\n\n"); + fprintf(self->dot_graph_file, "\"\n}\n\n"); } } @@ -1297,7 +1297,7 @@ TSParser *ts_parser_new() { self->stack = ts_stack_new(&self->tree_pool); self->finished_tree = NULL; self->reusable_node = reusable_node_new(); - self->print_debugging_graphs = false; + self->dot_graph_file = NULL; self->halt_on_error = false; ts_parser__set_cached_token(self, 0, NULL, NULL); return self; @@ -1345,8 +1345,8 @@ void ts_parser_set_logger(TSParser *self, TSLogger logger) { self->lexer.logger = logger; } -void ts_parser_print_debugging_graphs(TSParser *self, bool should_print) { - self->print_debugging_graphs = should_print; +void ts_parser_print_dot_graphs(TSParser *self, FILE *file) { + self->dot_graph_file = file; } void ts_parser_halt_on_error(TSParser *self, bool should_halt_on_error) { diff --git a/test/benchmarks.cc b/test/benchmarks.cc index d4f475b5..6612444e 100644 --- a/test/benchmarks.cc +++ b/test/benchmarks.cc @@ -46,7 +46,7 @@ int main(int argc, char *arg[]) { TSParser *parser = ts_parser_new(); if (getenv("TREE_SITTER_BENCHMARK_SVG")) { - ts_parser_print_debugging_graphs(parser, true); + ts_parser_print_dot_graphs(parser, stderr); } else if (getenv("TREE_SITTER_BENCHMARK_LOG")) { ts_parser_set_logger(parser, stderr_logger_new(false)); } diff --git a/test/integration/fuzzing-examples.cc b/test/integration/fuzzing-examples.cc index 92682d4c..aca95840 100644 --- a/test/integration/fuzzing-examples.cc +++ b/test/integration/fuzzing-examples.cc @@ -32,7 +32,7 @@ describe("examples found via fuzzing", [&]() { TSParser *parser = ts_parser_new(); if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) { - ts_parser_print_debugging_graphs(parser, true); + ts_parser_print_dot_graphs(parser, stderr); } const string &language_name = examples[i].first; diff --git a/test/integration/real_grammars.cc b/test/integration/real_grammars.cc index 0d594ce5..a628177f 100644 --- a/test/integration/real_grammars.cc +++ b/test/integration/real_grammars.cc @@ -43,7 +43,7 @@ for (auto &language_name : test_languages) { // ts_parser_set_logger(parser, stderr_logger_new(true)); if (debug_graphs_enabled) { - ts_parser_print_debugging_graphs(parser, true); + ts_parser_print_dot_graphs(parser, stderr); } }); diff --git a/test/integration/test_grammars.cc b/test/integration/test_grammars.cc index bba40ace..62174855 100644 --- a/test/integration/test_grammars.cc +++ b/test/integration/test_grammars.cc @@ -56,7 +56,7 @@ for (auto &language_name : test_languages) { ts_parser_set_language(parser, language); if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) { - ts_parser_print_debugging_graphs(parser, true); + ts_parser_print_dot_graphs(parser, stderr); } TSTree *tree = ts_parser_parse_string(parser, nullptr, entry.input.c_str(), entry.input.size()); diff --git a/test/runtime/parser_test.cc b/test/runtime/parser_test.cc index e9cfbe72..7d0b2d1d 100644 --- a/test/runtime/parser_test.cc +++ b/test/runtime/parser_test.cc @@ -27,7 +27,7 @@ describe("Parser", [&]() { tree = nullptr; parser = ts_parser_new(); if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) { - ts_parser_print_debugging_graphs(parser, true); + ts_parser_print_dot_graphs(parser, stderr); } }); From cd55d5275d1cce5129d62486ec445ad1760f1a3c Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 12:43:16 -0700 Subject: [PATCH 22/31] Update the readme to reflect the new API --- README.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 2feacc25..330dcf86 100644 --- a/README.md +++ b/README.md @@ -179,25 +179,28 @@ tokens, like `(` and `+`. This is useful when analyzing the meaning of a documen TSLanguage *tree_sitter_arithmetic(); int main() { - TSDocument *document = ts_document_new(); - ts_document_set_language(document, tree_sitter_arithmetic()); - ts_document_set_input_string(document, "a + b * 5"); - ts_document_parse(document); + TSParser *parser = ts_parser_new(); + ts_parser_set_language(parser, tree_sitter_arithmetic()); - TSNode root_node = ts_document_root_node(document); - assert(!strcmp(ts_node_type(root_node, document), "expression")); + const char *source_code = "a + b * 5"; + TSTree *tree = ts_parser_parse(parser, NULL, source_code, strlen(source_code)); + + TSNode root_node = ts_tree_root_node(tree); + assert(!strcmp(ts_node_type(root_node), "expression")); assert(ts_node_named_child_count(root_node) == 1); TSNode sum_node = ts_node_named_child(root_node, 0); - assert(!strcmp(ts_node_type(sum_node, document), "sum")); + assert(!strcmp(ts_node_type(sum_node), "sum")); assert(ts_node_named_child_count(sum_node) == 2); TSNode product_node = ts_node_child(ts_node_named_child(sum_node, 1), 0); - assert(!strcmp(ts_node_type(product_node, document), "product")); + assert(!strcmp(ts_node_type(product_node), "product")); assert(ts_node_named_child_count(product_node) == 2); - printf("Syntax tree: %s\n", ts_node_string(root_node, document)); - ts_document_free(document); + printf("Syntax tree: %s\n", ts_node_string(root_node)); + + ts_tree_delete(tree); + ts_parser_delete(parser); return 0; } ``` From f0c7295d272f383e04b4f418385106269683cb2a Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 12:47:49 -0700 Subject: [PATCH 23/31] Update fuzz driver to use new API --- test/fuzz/fuzzer.cc | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/test/fuzz/fuzzer.cc b/test/fuzz/fuzzer.cc index 7b1aea5b..570dfa7e 100644 --- a/test/fuzz/fuzzer.cc +++ b/test/fuzz/fuzzer.cc @@ -12,16 +12,15 @@ extern "C" const TSLanguage *TS_LANG(); extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { const char *str = reinterpret_cast(data); - TSDocument *document = ts_document_new(); - ts_document_set_language(document, TS_LANG()); - ts_document_set_input_string_with_length(document, str, size); + TSParser *parser = ts_parser_new(); + ts_parser_set_language(parser, TS_LANG()); + ts_parser_halt_on_error(parser, TS_HALT_ON_ERROR); - TSParseOptions options = {}; - options.halt_on_error = TS_HALT_ON_ERROR; - ts_document_parse_with_options(document, options); + TSTree *tree = ts_parser_parse_string(parser, NULL, str, size); + TSNode root_node = ts_document_root_node(tree); - TSNode root_node = ts_document_root_node(document); - ts_document_free(document); + ts_tree_delete(tree); + ts_parser_delete(parser); return 0; } From bf1bb1604fd1e667a35026cc803486299008aac4 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 12:57:41 -0700 Subject: [PATCH 24/31] Rename TSExternalTokenState -> ExternalScannerState --- include/tree_sitter/parser.h | 8 +++++--- src/runtime/parser.c | 10 +++++----- src/runtime/stack.c | 10 +++++----- src/runtime/subtree.c | 37 ++++++++++++++++++------------------ src/runtime/subtree.h | 10 +++++----- test/runtime/stack_test.cc | 12 ++++++------ 6 files changed, 45 insertions(+), 42 deletions(-) diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h index d99655f8..35e77a87 100644 --- a/include/tree_sitter/parser.h +++ b/include/tree_sitter/parser.h @@ -9,13 +9,14 @@ extern "C" { #include #include -typedef uint16_t TSSymbol; -typedef uint16_t TSStateId; - #define ts_builtin_sym_error ((TSSymbol)-1) #define ts_builtin_sym_end 0 #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 +typedef uint16_t TSSymbol; + +typedef uint16_t TSStateId; + typedef struct { bool visible : 1; bool named : 1; @@ -129,6 +130,7 @@ typedef struct TSLanguage { */ #define STATE(id) id + #define ACTIONS(id) id #define SHIFT(state_value) \ diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 02e96857..2115435e 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -247,8 +247,8 @@ static void ts_parser__restore_external_scanner(TSParser *self, Subtree *externa if (external_token) { self->language->external_scanner.deserialize( self->external_scanner_payload, - ts_external_token_state_data(&external_token->external_token_state), - external_token->external_token_state.length + ts_external_scanner_state_data(&external_token->external_scanner_state), + external_token->external_scanner_state.length ); } else { self->language->external_scanner.deserialize(self->external_scanner_payload, NULL, 0); @@ -391,7 +391,7 @@ static Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId p self->external_scanner_payload, self->lexer.debug_buffer ); - ts_external_token_state_init(&result->external_token_state, self->lexer.debug_buffer, length); + ts_external_scanner_state_init(&result->external_scanner_state, self->lexer.debug_buffer, length); } } @@ -408,7 +408,7 @@ static Subtree *ts_parser__get_cached_token(TSParser *self, size_t byte_index, TokenCache *cache = &self->token_cache; if (cache->token && cache->byte_index == byte_index && - ts_subtree_external_token_state_eq(cache->last_external_token, last_external_token)) { + ts_subtree_external_scanner_state_eq(cache->last_external_token, last_external_token)) { return cache->token; } else { return NULL; @@ -464,7 +464,7 @@ static Subtree *ts_parser__get_lookahead(TSParser *self, StackVersion version, T continue; } - if (!ts_subtree_external_token_state_eq(reusable_node->last_external_token, last_external_token)) { + if (!ts_subtree_external_scanner_state_eq(reusable_node->last_external_token, last_external_token)) { LOG("reusable_node_has_different_external_scanner_state symbol:%s", SYM_NAME(result->symbol)); reusable_node_advance(reusable_node); continue; diff --git a/src/runtime/stack.c b/src/runtime/stack.c index c8af7a58..21468e49 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -164,7 +164,7 @@ static bool stack__subtree_is_equivalent(const Subtree *left, const Subtree *rig left->padding.bytes == right->padding.bytes && left->size.bytes == right->size.bytes && left->extra == right->extra && - ts_subtree_external_token_state_eq(left, right)))); + ts_subtree_external_scanner_state_eq(left, right)))); } static void stack_node_add_link(StackNode *self, StackLink link) { @@ -605,7 +605,7 @@ bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version head1->node->state == head2->node->state && head1->node->position.bytes == head2->node->position.bytes && head1->node->error_cost == head2->node->error_cost && - ts_subtree_external_token_state_eq(head1->last_external_token, head2->last_external_token); + ts_subtree_external_scanner_state_eq(head1->last_external_token, head2->last_external_token); } void ts_stack_halt(Stack *self, StackVersion version) { @@ -684,9 +684,9 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) ); if (head->last_external_token) { - TSExternalTokenState *state = &head->last_external_token->external_token_state; - const char *data = ts_external_token_state_data(state); - fprintf(f, "\nexternal_token_state:"); + ExternalScannerState *state = &head->last_external_token->external_scanner_state; + const char *data = ts_external_scanner_state_data(state); + fprintf(f, "\nexternal_scanner_state:"); for (uint32_t j = 0; j < state->length; j++) fprintf(f, " %2X", data[j]); } diff --git a/src/runtime/subtree.c b/src/runtime/subtree.c index aff2a6fc..c3da95c5 100644 --- a/src/runtime/subtree.c +++ b/src/runtime/subtree.c @@ -21,27 +21,27 @@ TSStateId TS_TREE_STATE_NONE = USHRT_MAX; static const uint32_t MAX_TREE_POOL_SIZE = 1024; -static const TSExternalTokenState empty_state = {.length = 0, .short_data = {0}}; +static const ExternalScannerState empty_state = {.length = 0, .short_data = {0}}; -// ExternalTokenState +// ExternalScannerState -void ts_external_token_state_init(TSExternalTokenState *self, const char *content, unsigned length) { +void ts_external_scanner_state_init(ExternalScannerState *self, const char *data, unsigned length) { self->length = length; if (length > sizeof(self->short_data)) { self->long_data = ts_malloc(length); - memcpy(self->long_data, content, length); + memcpy(self->long_data, data, length); } else { - memcpy(self->short_data, content, length); + memcpy(self->short_data, data, length); } } -void ts_external_token_state_delete(TSExternalTokenState *self) { +void ts_external_scanner_state_delete(ExternalScannerState *self) { if (self->length > sizeof(self->short_data)) { ts_free(self->long_data); } } -const char *ts_external_token_state_data(const TSExternalTokenState *self) { +const char *ts_external_scanner_state_data(const ExternalScannerState *self) { if (self->length > sizeof(self->short_data)) { return self->long_data; } else { @@ -49,10 +49,11 @@ const char *ts_external_token_state_data(const TSExternalTokenState *self) { } } -bool ts_external_token_state_eq(const TSExternalTokenState *a, const TSExternalTokenState *b) { - return a == b || - (a->length == b->length && - memcmp(ts_external_token_state_data(a), ts_external_token_state_data(b), a->length) == 0); +bool ts_external_scanner_state_eq(const ExternalScannerState *a, const ExternalScannerState *b) { + return a == b || ( + a->length == b->length && + !memcmp(ts_external_scanner_state_data(a), ts_external_scanner_state_data(b), a->length) + ); } // SubtreeArray @@ -403,7 +404,7 @@ void ts_subtree_release(SubtreePool *pool, Subtree *self) { } array_delete(&tree->children); } else if (tree->has_external_tokens) { - ts_external_token_state_delete(&tree->external_token_state); + ts_external_scanner_state_delete(&tree->external_scanner_state); } ts_subtree_pool_free(pool, tree); } @@ -694,10 +695,10 @@ void ts_subtree_print_dot_graph(const Subtree *self, const TSLanguage *language, fprintf(f, "}\n"); } -bool ts_subtree_external_token_state_eq(const Subtree *self, const Subtree *other) { - const TSExternalTokenState *state1 = &empty_state; - const TSExternalTokenState *state2 = &empty_state; - if (self && self->has_external_tokens) state1 = &self->external_token_state; - if (other && other->has_external_tokens) state2 = &other->external_token_state; - return ts_external_token_state_eq(state1, state2); +bool ts_subtree_external_scanner_state_eq(const Subtree *self, const Subtree *other) { + const ExternalScannerState *state1 = &empty_state; + const ExternalScannerState *state2 = &empty_state; + if (self && self->has_external_tokens) state1 = &self->external_scanner_state; + if (other && other->has_external_tokens) state2 = &other->external_scanner_state; + return ts_external_scanner_state_eq(state1, state2); } diff --git a/src/runtime/subtree.h b/src/runtime/subtree.h index fd1fcaff..2f7d60b0 100644 --- a/src/runtime/subtree.h +++ b/src/runtime/subtree.h @@ -20,7 +20,7 @@ typedef struct { char short_data[sizeof(char *) + sizeof(uint32_t)]; }; uint32_t length; -} TSExternalTokenState; +} ExternalScannerState; typedef struct Subtree Subtree; @@ -61,7 +61,7 @@ struct Subtree { }; struct { uint32_t _2; - TSExternalTokenState external_token_state; + ExternalScannerState external_scanner_state; }; struct { uint32_t _1; @@ -75,8 +75,8 @@ typedef struct { SubtreeArray tree_stack; } SubtreePool; -void ts_external_token_state_init(TSExternalTokenState *, const char *, unsigned); -const char *ts_external_token_state_data(const TSExternalTokenState *); +void ts_external_scanner_state_init(ExternalScannerState *, const char *, unsigned); +const char *ts_external_scanner_state_data(const ExternalScannerState *); bool ts_subtree_array_copy(SubtreeArray, SubtreeArray *); void ts_subtree_array_delete(SubtreePool *, SubtreeArray *); @@ -104,7 +104,7 @@ Subtree *ts_subtree_edit(Subtree *, const TSInputEdit *edit, SubtreePool *); char *ts_subtree_string(const Subtree *, const TSLanguage *, bool include_all); void ts_subtree_print_dot_graph(const Subtree *, const TSLanguage *, FILE *); Subtree *ts_subtree_last_external_token(Subtree *); -bool ts_subtree_external_token_state_eq(const Subtree *, const Subtree *); +bool ts_subtree_external_scanner_state_eq(const Subtree *, const Subtree *); static inline uint32_t ts_subtree_total_bytes(const Subtree *self) { return self->padding.bytes + self->size.bytes; diff --git a/test/runtime/stack_test.cc b/test/runtime/stack_test.cc index ae5b1589..a6ec09c4 100644 --- a/test/runtime/stack_test.cc +++ b/test/runtime/stack_test.cc @@ -528,8 +528,8 @@ describe("Stack", [&]() { before_each([&]() { subtrees[1]->has_external_tokens = true; subtrees[2]->has_external_tokens = true; - ts_external_token_state_init(&subtrees[1]->external_token_state, NULL, 0); - ts_external_token_state_init(&subtrees[2]->external_token_state, NULL, 0); + ts_external_scanner_state_init(&subtrees[1]->external_scanner_state, NULL, 0); + ts_external_scanner_state_init(&subtrees[2]->external_scanner_state, NULL, 0); }); it("allows the state to be retrieved", [&]() { @@ -546,8 +546,8 @@ describe("Stack", [&]() { }); it("does not merge stack versions with different external token states", [&]() { - ts_external_token_state_init(&subtrees[1]->external_token_state, "abcd", 2); - ts_external_token_state_init(&subtrees[2]->external_token_state, "ABCD", 2); + ts_external_scanner_state_init(&subtrees[1]->external_scanner_state, "abcd", 2); + ts_external_scanner_state_init(&subtrees[2]->external_scanner_state, "ABCD", 2); ts_stack_copy_version(stack, 0); push(0, subtrees[0], 5); @@ -560,8 +560,8 @@ describe("Stack", [&]() { }); it("merges stack versions with identical external token states", [&]() { - ts_external_token_state_init(&subtrees[1]->external_token_state, "abcd", 2); - ts_external_token_state_init(&subtrees[2]->external_token_state, "abcd", 2); + ts_external_scanner_state_init(&subtrees[1]->external_scanner_state, "abcd", 2); + ts_external_scanner_state_init(&subtrees[2]->external_scanner_state, "abcd", 2); ts_stack_copy_version(stack, 0); push(0, subtrees[0], 5); From 20c183b7cd7f33a61923b647ddd238f8721db748 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 13:02:12 -0700 Subject: [PATCH 25/31] Rename ts_subtree_make_* -> ts_subtree_new_* --- src/runtime/parser.c | 26 +++++++-------- src/runtime/subtree.c | 28 ++++++++-------- src/runtime/subtree.h | 16 ++++----- test/runtime/stack_test.cc | 2 +- test/runtime/subtree_test.cc | 64 ++++++++++++++++++------------------ 5 files changed, 66 insertions(+), 70 deletions(-) diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 2115435e..8332501b 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -358,7 +358,7 @@ static Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId p if (skipped_error) { Length padding = length_sub(error_start_position, start_position); Length size = length_sub(error_end_position, error_start_position); - result = ts_subtree_make_error(&self->tree_pool, size, padding, first_error_character, self->language); + result = ts_subtree_new_error(&self->tree_pool, size, padding, first_error_character, self->language); } else { if (self->lexer.token_end_position.bytes < self->lexer.token_start_position.bytes) { self->lexer.token_start_position = self->lexer.token_end_position; @@ -383,7 +383,7 @@ static Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId p } } - result = ts_subtree_make_leaf(&self->tree_pool, symbol, padding, size, self->language); + result = ts_subtree_new_leaf(&self->tree_pool, symbol, padding, size, self->language); if (found_external_token) { result->has_external_tokens = true; @@ -577,7 +577,7 @@ static void ts_parser__shift(TSParser *self, StackVersion version, TSStateId sta Subtree *lookahead, bool extra) { if (extra != lookahead->extra) { if (ts_stack_version_count(self->stack) > 1) { - lookahead = ts_subtree_make_copy(&self->tree_pool, lookahead); + lookahead = ts_subtree_new_copy(&self->tree_pool, lookahead); } else { ts_subtree_retain(lookahead); } @@ -625,7 +625,7 @@ static StackSliceArray ts_parser__reduce(TSParser *self, StackVersion version, T children.size--; } - Subtree *parent = ts_subtree_make_node(&self->tree_pool, + Subtree *parent = ts_subtree_new_node(&self->tree_pool, symbol, &children, alias_sequence_id, self->language ); @@ -735,7 +735,7 @@ static void ts_parser__accept(TSParser *self, StackVersion version, Subtree *loo ts_subtree_retain(child->children.contents[k]); } array_splice(&trees, j, 1, &child->children); - root = ts_subtree_make_node( + root = ts_subtree_new_node( &self->tree_pool, child->symbol, &trees, child->alias_sequence_id, self->language ); @@ -875,7 +875,7 @@ static void ts_parser__handle_error(TSParser *self, StackVersion version, lookahead_symbol )) { StackVersion version_with_missing_tree = ts_stack_copy_version(self->stack, v); - Subtree *missing_tree = ts_subtree_make_missing_leaf(&self->tree_pool, missing_symbol, self->language); + Subtree *missing_tree = ts_subtree_new_missing_leaf(&self->tree_pool, missing_symbol, self->language); ts_stack_push( self->stack, version_with_missing_tree, missing_tree, false, @@ -920,15 +920,15 @@ static void ts_parser__halt_parse(TSParser *self) { ts_stack_position(self->stack, 0) ); - Subtree *filler_node = ts_subtree_make_error(&self->tree_pool, remaining_length, length_zero(), 0, self->language); + Subtree *filler_node = ts_subtree_new_error(&self->tree_pool, remaining_length, length_zero(), 0, self->language); filler_node->visible = false; ts_stack_push(self->stack, 0, filler_node, false, 0); SubtreeArray children = array_new(); - Subtree *root_error = ts_subtree_make_error_node(&self->tree_pool, &children, self->language); + Subtree *root_error = ts_subtree_new_error_node(&self->tree_pool, &children, self->language); ts_stack_push(self->stack, 0, root_error, false, 0); - Subtree *eof = ts_subtree_make_leaf(&self->tree_pool, ts_builtin_sym_end, length_zero(), length_zero(), self->language); + Subtree *eof = ts_subtree_new_leaf(&self->tree_pool, ts_builtin_sym_end, length_zero(), length_zero(), self->language); ts_parser__accept(self, 0, eof); ts_subtree_release(&self->tree_pool, eof); } @@ -967,7 +967,7 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, un SubtreeArray trailing_extras = ts_subtree_array_remove_trailing_extras(&slice.subtrees); if (slice.subtrees.size > 0) { - Subtree *error = ts_subtree_make_error_node(&self->tree_pool, &slice.subtrees, self->language); + Subtree *error = ts_subtree_new_error_node(&self->tree_pool, &slice.subtrees, self->language); error->extra = true; ts_stack_push(self->stack, slice.version, error, false, goal_state); } else { @@ -1048,7 +1048,7 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree *lo if (lookahead->symbol == ts_builtin_sym_end) { LOG("recover_eof"); SubtreeArray children = array_new(); - Subtree *parent = ts_subtree_make_error_node(&self->tree_pool, &children, self->language); + Subtree *parent = ts_subtree_new_error_node(&self->tree_pool, &children, self->language); ts_stack_push(self->stack, version, parent, false, 1); ts_parser__accept(self, version, lookahead); return; @@ -1075,7 +1075,7 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree *lo SubtreeArray children = array_new(); array_reserve(&children, 1); array_push(&children, lookahead); - Subtree *error_repeat = ts_subtree_make_node( + Subtree *error_repeat = ts_subtree_new_node( &self->tree_pool, ts_builtin_sym_error_repeat, &children, @@ -1089,7 +1089,7 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree *lo assert(pop.contents[0].subtrees.size == 1); ts_stack_renumber_version(self->stack, pop.contents[0].version, version); array_push(&pop.contents[0].subtrees, error_repeat); - error_repeat = ts_subtree_make_node( + error_repeat = ts_subtree_new_node( &self->tree_pool, ts_builtin_sym_error_repeat, &pop.contents[0].subtrees, diff --git a/src/runtime/subtree.c b/src/runtime/subtree.c index c3da95c5..0cd125bf 100644 --- a/src/runtime/subtree.c +++ b/src/runtime/subtree.c @@ -141,7 +141,7 @@ void ts_subtree_pool_free(SubtreePool *self, Subtree *tree) { // Subtree -Subtree *ts_subtree_make_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, Length size, +Subtree *ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, Length size, const TSLanguage *language) { TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); Subtree *result = ts_subtree_pool_allocate(pool); @@ -166,16 +166,16 @@ Subtree *ts_subtree_make_leaf(SubtreePool *pool, TSSymbol symbol, Length padding return result; } -Subtree *ts_subtree_make_error(SubtreePool *pool, Length size, Length padding, +Subtree *ts_subtree_new_error(SubtreePool *pool, Length size, Length padding, int32_t lookahead_char, const TSLanguage *language) { - Subtree *result = ts_subtree_make_leaf(pool, ts_builtin_sym_error, padding, size, language); + Subtree *result = ts_subtree_new_leaf(pool, ts_builtin_sym_error, padding, size, language); result->fragile_left = true; result->fragile_right = true; result->lookahead_char = lookahead_char; return result; } -Subtree *ts_subtree_make_copy(SubtreePool *pool, Subtree *self) { +Subtree *ts_subtree_new_copy(SubtreePool *pool, Subtree *self) { Subtree *result = ts_subtree_pool_allocate(pool); *result = *self; if (result->children.size > 0) { @@ -185,11 +185,11 @@ Subtree *ts_subtree_make_copy(SubtreePool *pool, Subtree *self) { return result; } -static Subtree *ts_subtree_make_mut(SubtreePool *pool, Subtree *self) { +static Subtree *ts_subtree_new_mut(SubtreePool *pool, Subtree *self) { if (self->ref_count == 1) { return self; } else { - Subtree *result = ts_subtree_make_copy(pool, self); + Subtree *result = ts_subtree_new_copy(pool, self); ts_subtree_release(pool, self); return result; } @@ -360,9 +360,9 @@ void ts_subtree_set_children(Subtree *self, SubtreeArray *children, const TSLang } } -Subtree *ts_subtree_make_node(SubtreePool *pool, TSSymbol symbol, SubtreeArray *children, +Subtree *ts_subtree_new_node(SubtreePool *pool, TSSymbol symbol, SubtreeArray *children, unsigned alias_sequence_id, const TSLanguage *language) { - Subtree *result = ts_subtree_make_leaf(pool, symbol, length_zero(), length_zero(), language); + Subtree *result = ts_subtree_new_leaf(pool, symbol, length_zero(), length_zero(), language); result->alias_sequence_id = alias_sequence_id; if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { result->fragile_left = true; @@ -372,14 +372,14 @@ Subtree *ts_subtree_make_node(SubtreePool *pool, TSSymbol symbol, SubtreeArray * return result; } -Subtree *ts_subtree_make_error_node(SubtreePool *pool, SubtreeArray *children, +Subtree *ts_subtree_new_error_node(SubtreePool *pool, SubtreeArray *children, const TSLanguage *language) { - return ts_subtree_make_node(pool, ts_builtin_sym_error, children, 0, language); + return ts_subtree_new_node(pool, ts_builtin_sym_error, children, 0, language); } -Subtree *ts_subtree_make_missing_leaf(SubtreePool *pool, TSSymbol symbol, +Subtree *ts_subtree_new_missing_leaf(SubtreePool *pool, TSSymbol symbol, const TSLanguage *language) { - Subtree *result = ts_subtree_make_leaf(pool, symbol, length_zero(), length_zero(), language); + Subtree *result = ts_subtree_new_leaf(pool, symbol, length_zero(), length_zero(), language); result->is_missing = true; result->error_cost = ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY; return result; @@ -464,7 +464,7 @@ Subtree *ts_subtree_invalidate_lookahead(Subtree *self, uint32_t edit_byte_offse SubtreePool *pool) { if (edit_byte_offset >= self->bytes_scanned) return self; - Subtree *result = ts_subtree_make_mut(pool, self); + Subtree *result = ts_subtree_new_mut(pool, self); result->has_changes = true; if (result->children.size > 0) { @@ -484,7 +484,7 @@ Subtree *ts_subtree__edit(Subtree *self, Edit edit, SubtreePool *pool) { Length new_end = length_add(edit.start, edit.added); Length old_end = length_add(edit.start, edit.removed); - Subtree *result = ts_subtree_make_mut(pool, self); + Subtree *result = ts_subtree_new_mut(pool, self); result->has_changes = true; if (old_end.bytes <= result->padding.bytes) { diff --git a/src/runtime/subtree.h b/src/runtime/subtree.h index 2f7d60b0..da69383d 100644 --- a/src/runtime/subtree.h +++ b/src/runtime/subtree.h @@ -88,12 +88,12 @@ void ts_subtree_pool_delete(SubtreePool *); Subtree *ts_subtree_pool_allocate(SubtreePool *); void ts_subtree_pool_free(SubtreePool *, Subtree *); -Subtree *ts_subtree_make_leaf(SubtreePool *, TSSymbol, Length, Length, const TSLanguage *); -Subtree *ts_subtree_make_node(SubtreePool *, TSSymbol, SubtreeArray *, unsigned, const TSLanguage *); -Subtree *ts_subtree_make_copy(SubtreePool *, Subtree *child); -Subtree *ts_subtree_make_error_node(SubtreePool *, SubtreeArray *, const TSLanguage *); -Subtree *ts_subtree_make_error(SubtreePool *, Length, Length, int32_t, const TSLanguage *); -Subtree *ts_subtree_make_missing_leaf(SubtreePool *, TSSymbol, const TSLanguage *); +Subtree *ts_subtree_new_leaf(SubtreePool *, TSSymbol, Length, Length, const TSLanguage *); +Subtree *ts_subtree_new_node(SubtreePool *, TSSymbol, SubtreeArray *, unsigned, const TSLanguage *); +Subtree *ts_subtree_new_copy(SubtreePool *, Subtree *child); +Subtree *ts_subtree_new_error_node(SubtreePool *, SubtreeArray *, const TSLanguage *); +Subtree *ts_subtree_new_error(SubtreePool *, Length, Length, int32_t, const TSLanguage *); +Subtree *ts_subtree_new_missing_leaf(SubtreePool *, TSSymbol, const TSLanguage *); void ts_subtree_retain(Subtree *tree); void ts_subtree_release(SubtreePool *, Subtree *tree); bool ts_subtree_eq(const Subtree *tree1, const Subtree *tree2); @@ -110,10 +110,6 @@ static inline uint32_t ts_subtree_total_bytes(const Subtree *self) { return self->padding.bytes + self->size.bytes; } -static inline uint32_t ts_subtree_total_rows(const Subtree *self) { - return self->padding.extent.row + self->size.extent.row; -} - static inline Length ts_subtree_total_size(const Subtree *self) { return length_add(self->padding, self->size); } diff --git a/test/runtime/stack_test.cc b/test/runtime/stack_test.cc index a6ec09c4..67812a07 100644 --- a/test/runtime/stack_test.cc +++ b/test/runtime/stack_test.cc @@ -84,7 +84,7 @@ describe("Stack", [&]() { dummy_language.symbol_metadata = symbol_metadata; for (size_t i = 0; i < subtree_count; i++) { - subtrees[i] = ts_subtree_make_leaf(&pool, i, length_zero(), tree_len, &dummy_language); + subtrees[i] = ts_subtree_new_leaf(&pool, i, length_zero(), tree_len, &dummy_language); } }); diff --git a/test/runtime/subtree_test.cc b/test/runtime/subtree_test.cc index 093e68db..4d32ec32 100644 --- a/test/runtime/subtree_test.cc +++ b/test/runtime/subtree_test.cc @@ -50,7 +50,7 @@ describe("Subtree", []() { describe("make_leaf", [&]() { it("does not mark the tree as fragile", [&]() { - Subtree *tree = ts_subtree_make_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); + Subtree *tree = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); AssertThat(tree->fragile_left, IsFalse()); AssertThat(tree->fragile_right, IsFalse()); @@ -60,7 +60,7 @@ describe("Subtree", []() { describe("make_error", [&]() { it("marks the tree as fragile", [&]() { - Subtree *error_tree = ts_subtree_make_error( + Subtree *error_tree = ts_subtree_new_error( &pool, length_zero(), length_zero(), @@ -79,12 +79,12 @@ describe("Subtree", []() { Subtree *tree1, *tree2, *parent1; before_each([&]() { - tree1 = ts_subtree_make_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); - tree2 = ts_subtree_make_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); + tree1 = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); + tree2 = ts_subtree_new_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); ts_subtree_retain(tree1); ts_subtree_retain(tree2); - parent1 = ts_subtree_make_node(&pool, symbol3, tree_array({ + parent1 = ts_subtree_new_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); @@ -112,7 +112,7 @@ describe("Subtree", []() { ts_subtree_retain(tree1); ts_subtree_retain(tree2); - parent = ts_subtree_make_node(&pool, symbol3, tree_array({ + parent = ts_subtree_new_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); @@ -136,7 +136,7 @@ describe("Subtree", []() { ts_subtree_retain(tree1); ts_subtree_retain(tree2); - parent = ts_subtree_make_node(&pool, symbol3, tree_array({ + parent = ts_subtree_new_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); @@ -160,7 +160,7 @@ describe("Subtree", []() { ts_subtree_retain(tree1); ts_subtree_retain(tree2); - parent = ts_subtree_make_node(&pool, symbol3, tree_array({ + parent = ts_subtree_new_node(&pool, symbol3, tree_array({ tree1, tree2, }), 0, &language); @@ -181,10 +181,10 @@ describe("Subtree", []() { Subtree *tree; before_each([&]() { - tree = ts_subtree_make_node(&pool, symbol1, tree_array({ - ts_subtree_make_leaf(&pool, symbol2, {2, {0, 2}}, {3, {0, 3}}, &language), - ts_subtree_make_leaf(&pool, symbol3, {2, {0, 2}}, {3, {0, 3}}, &language), - ts_subtree_make_leaf(&pool, symbol4, {2, {0, 2}}, {3, {0, 3}}, &language), + tree = ts_subtree_new_node(&pool, symbol1, tree_array({ + ts_subtree_new_leaf(&pool, symbol2, {2, {0, 2}}, {3, {0, 3}}, &language), + ts_subtree_new_leaf(&pool, symbol3, {2, {0, 2}}, {3, {0, 3}}, &language), + ts_subtree_new_leaf(&pool, symbol4, {2, {0, 2}}, {3, {0, 3}}, &language), }), 0, &language); AssertThat(tree->padding, Equals({2, {0, 2}})); @@ -373,7 +373,7 @@ describe("Subtree", []() { Subtree *leaf; before_each([&]() { - leaf = ts_subtree_make_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); + leaf = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); }); after_each([&]() { @@ -381,17 +381,17 @@ describe("Subtree", []() { }); it("returns true for identical trees", [&]() { - Subtree *leaf_copy = ts_subtree_make_leaf(&pool, symbol1, {2, {1, 1}}, {5, {1, 4}}, &language); + Subtree *leaf_copy = ts_subtree_new_leaf(&pool, symbol1, {2, {1, 1}}, {5, {1, 4}}, &language); AssertThat(ts_subtree_eq(leaf, leaf_copy), IsTrue()); - Subtree *parent = ts_subtree_make_node(&pool, symbol2, tree_array({ + Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({ leaf, leaf_copy, }), 0, &language); ts_subtree_retain(leaf); ts_subtree_retain(leaf_copy); - Subtree *parent_copy = ts_subtree_make_node(&pool, symbol2, tree_array({ + Subtree *parent_copy = ts_subtree_new_node(&pool, symbol2, tree_array({ leaf, leaf_copy, }), 0, &language); @@ -406,7 +406,7 @@ describe("Subtree", []() { }); it("returns false for trees with different symbols", [&]() { - Subtree *different_leaf = ts_subtree_make_leaf( + Subtree *different_leaf = ts_subtree_new_leaf( &pool, leaf->symbol + 1, leaf->padding, @@ -419,33 +419,33 @@ describe("Subtree", []() { }); it("returns false for trees with different options", [&]() { - Subtree *different_leaf = ts_subtree_make_leaf(&pool, leaf->symbol, leaf->padding, leaf->size, &language); + Subtree *different_leaf = ts_subtree_new_leaf(&pool, leaf->symbol, leaf->padding, leaf->size, &language); different_leaf->visible = !leaf->visible; AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); ts_subtree_release(&pool, different_leaf); }); it("returns false for trees with different paddings or sizes", [&]() { - Subtree *different_leaf = ts_subtree_make_leaf(&pool, leaf->symbol, {}, leaf->size, &language); + Subtree *different_leaf = ts_subtree_new_leaf(&pool, leaf->symbol, {}, leaf->size, &language); AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); ts_subtree_release(&pool, different_leaf); - different_leaf = ts_subtree_make_leaf(&pool, symbol1, leaf->padding, {}, &language); + different_leaf = ts_subtree_new_leaf(&pool, symbol1, leaf->padding, {}, &language); AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); ts_subtree_release(&pool, different_leaf); }); it("returns false for trees with different children", [&]() { - Subtree *leaf2 = ts_subtree_make_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); + Subtree *leaf2 = ts_subtree_new_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); - Subtree *parent = ts_subtree_make_node(&pool, symbol2, tree_array({ + Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({ leaf, leaf2, }), 0, &language); ts_subtree_retain(leaf); ts_subtree_retain(leaf2); - Subtree *different_parent = ts_subtree_make_node(&pool, symbol2, tree_array({ + Subtree *different_parent = ts_subtree_new_node(&pool, symbol2, tree_array({ leaf2, leaf, }), 0, &language); @@ -473,17 +473,17 @@ describe("Subtree", []() { it("returns the last serialized external token state in the given tree", [&]() { Subtree *tree1, *tree2, *tree3, *tree4, *tree5, *tree6, *tree7, *tree8, *tree9; - tree1 = ts_subtree_make_node(&pool, symbol1, tree_array({ - (tree2 = ts_subtree_make_node(&pool, symbol2, tree_array({ - (tree3 = make_external(ts_subtree_make_leaf(&pool, symbol3, padding, size, &language))), - (tree4 = ts_subtree_make_leaf(&pool, symbol4, padding, size, &language)), - (tree5 = ts_subtree_make_leaf(&pool, symbol5, padding, size, &language)), + tree1 = ts_subtree_new_node(&pool, symbol1, tree_array({ + (tree2 = ts_subtree_new_node(&pool, symbol2, tree_array({ + (tree3 = make_external(ts_subtree_new_leaf(&pool, symbol3, padding, size, &language))), + (tree4 = ts_subtree_new_leaf(&pool, symbol4, padding, size, &language)), + (tree5 = ts_subtree_new_leaf(&pool, symbol5, padding, size, &language)), }), 0, &language)), - (tree6 = ts_subtree_make_node(&pool, symbol6, tree_array({ - (tree7 = ts_subtree_make_node(&pool, symbol7, tree_array({ - (tree8 = ts_subtree_make_leaf(&pool, symbol8, padding, size, &language)), + (tree6 = ts_subtree_new_node(&pool, symbol6, tree_array({ + (tree7 = ts_subtree_new_node(&pool, symbol7, tree_array({ + (tree8 = ts_subtree_new_leaf(&pool, symbol8, padding, size, &language)), }), 0, &language)), - (tree9 = ts_subtree_make_leaf(&pool, symbol9, padding, size, &language)), + (tree9 = ts_subtree_new_leaf(&pool, symbol9, padding, size, &language)), }), 0, &language)), }), 0, &language); From 6bb63f549f87183a8538aa7ca4224379c9d25b09 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 14:59:59 -0700 Subject: [PATCH 26/31] Fix unused lambda captures --- src/compiler/build_tables/lex_item_transitions.cc | 2 +- src/compiler/prepare_grammar/extract_tokens.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/compiler/build_tables/lex_item_transitions.cc b/src/compiler/build_tables/lex_item_transitions.cc index d30bf011..7b4eb611 100644 --- a/src/compiler/build_tables/lex_item_transitions.cc +++ b/src/compiler/build_tables/lex_item_transitions.cc @@ -80,7 +80,7 @@ class TransitionBuilder { public: void apply(const Rule &rule) { rule.match( - [this](const rules::Blank &) {}, + [](const rules::Blank &) {}, [this](const rules::CharacterSet &character_set) { PrecedenceRange precedence; diff --git a/src/compiler/prepare_grammar/extract_tokens.cc b/src/compiler/prepare_grammar/extract_tokens.cc index 2e5cf1d9..93b06be2 100644 --- a/src/compiler/prepare_grammar/extract_tokens.cc +++ b/src/compiler/prepare_grammar/extract_tokens.cc @@ -30,7 +30,7 @@ class SymbolReplacer { Rule apply(const Rule &rule) { return rule.match( - [this](const rules::Blank &blank) -> Rule { + [](const rules::Blank &blank) -> Rule { return blank; }, @@ -110,7 +110,7 @@ class TokenExtractor { public: Rule apply(const rules::Rule &rule) { return rule.match( - [this](const rules::Blank &blank) -> Rule { return blank; }, + [](const rules::Blank &blank) -> Rule { return blank; }, [this](const rules::Metadata &rule) -> Rule { if (rule.params.is_token) { From fe5350617546e26403a6e6b8f8da5068b22959a2 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 15:06:13 -0700 Subject: [PATCH 27/31] Declare subtrees as const wherever possible Co-Authored-By: Rick Winfrey --- include/tree_sitter/runtime.h | 2 +- src/runtime/get_changed_ranges.c | 22 +++--- src/runtime/get_changed_ranges.h | 3 +- src/runtime/node.c | 2 +- src/runtime/parser.c | 108 +++++++++++++------------- src/runtime/reusable_node.h | 8 +- src/runtime/stack.c | 17 ++-- src/runtime/stack.h | 6 +- src/runtime/subtree.c | 129 +++++++++++++++++-------------- src/runtime/subtree.h | 20 ++--- src/runtime/tree.c | 2 +- src/runtime/tree_cursor.c | 8 +- src/runtime/tree_cursor.h | 2 +- test/helpers/tree_helpers.cc | 6 +- test/helpers/tree_helpers.h | 4 +- test/runtime/stack_test.cc | 60 +++++++------- test/runtime/subtree_test.cc | 67 +++++++++------- 17 files changed, 244 insertions(+), 222 deletions(-) diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index acd2c3b3..8f379718 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -85,7 +85,7 @@ TSTree *ts_parser_parse(TSParser *, const TSTree *, TSInput); TSTree *ts_parser_parse_string(TSParser *, const TSTree *, const char *, uint32_t); TSTree *ts_tree_copy(const TSTree *); -void ts_tree_delete(const TSTree *); +void ts_tree_delete(TSTree *); TSNode ts_tree_root_node(const TSTree *); void ts_tree_edit(TSTree *, const TSInputEdit *); TSRange *ts_tree_get_changed_ranges(const TSTree *, const TSTree *, uint32_t *); diff --git a/src/runtime/get_changed_ranges.c b/src/runtime/get_changed_ranges.c index 032fdaab..2b015d27 100644 --- a/src/runtime/get_changed_ranges.c +++ b/src/runtime/get_changed_ranges.c @@ -31,7 +31,7 @@ typedef struct { bool in_padding; } Iterator; -static Iterator iterator_new(TSTreeCursor *cursor, Subtree *tree, const TSLanguage *language) { +static Iterator iterator_new(TSTreeCursor *cursor, const Subtree *tree, const TSLanguage *language) { array_clear(&cursor->stack); array_push(&cursor->stack, ((TreeCursorEntry){ .subtree = tree, @@ -74,14 +74,14 @@ static bool iterator_tree_is_visible(const Iterator *self) { TreeCursorEntry entry = *array_back(&self->cursor.stack); if (entry.subtree->visible) return true; if (self->cursor.stack.size > 1) { - Subtree *parent = self->cursor.stack.contents[self->cursor.stack.size - 2].subtree; + const Subtree *parent = self->cursor.stack.contents[self->cursor.stack.size - 2].subtree; const TSSymbol *alias_sequence = ts_language_alias_sequence(self->language, parent->alias_sequence_id); return alias_sequence && alias_sequence[entry.structural_child_index] != 0; } return false; } -static void iterator_get_visible_state(const Iterator *self, Subtree **tree, +static void iterator_get_visible_state(const Iterator *self, const Subtree **tree, TSSymbol *alias_symbol, uint32_t *start_byte) { uint32_t i = self->cursor.stack.size - 1; @@ -94,7 +94,7 @@ static void iterator_get_visible_state(const Iterator *self, Subtree **tree, TreeCursorEntry entry = self->cursor.stack.contents[i]; if (i > 0) { - Subtree *parent = self->cursor.stack.contents[i - 1].subtree; + const Subtree *parent = self->cursor.stack.contents[i - 1].subtree; const TSSymbol *alias_sequence = ts_language_alias_sequence( self->language, parent->alias_sequence_id @@ -129,7 +129,7 @@ static bool iterator_descend(Iterator *self, uint32_t goal_position) { Length position = entry.position; uint32_t structural_child_index = 0; for (uint32_t i = 0; i < entry.subtree->children.size; i++) { - Subtree *child = entry.subtree->children.contents[i]; + const Subtree *child = entry.subtree->children.contents[i]; Length child_left = length_add(position, child->padding); Length child_right = length_add(child_left, child->size); @@ -178,13 +178,13 @@ static void iterator_advance(Iterator *self) { TreeCursorEntry entry = array_pop(&self->cursor.stack); if (iterator_done(self)) return; - Subtree *parent = array_back(&self->cursor.stack)->subtree; + const Subtree *parent = array_back(&self->cursor.stack)->subtree; uint32_t child_index = entry.child_index + 1; if (parent->children.size > child_index) { Length position = length_add(entry.position, ts_subtree_total_size(entry.subtree)); uint32_t structural_child_index = entry.structural_child_index; if (!entry.subtree->extra) structural_child_index++; - Subtree *next_child = parent->children.contents[child_index]; + const Subtree *next_child = parent->children.contents[child_index]; array_push(&self->cursor.stack, ((TreeCursorEntry){ .subtree = next_child, @@ -214,7 +214,7 @@ typedef enum { } IteratorComparison; IteratorComparison iterator_compare(const Iterator *old_iter, const Iterator *new_iter) { - Subtree *old_tree = NULL, *new_tree = NULL; + const Subtree *old_tree = NULL, *new_tree = NULL; uint32_t old_start = 0, new_start = 0; TSSymbol old_alias_symbol = 0, new_alias_symbol = 0; iterator_get_visible_state(old_iter, &old_tree, &old_alias_symbol, &old_start); @@ -261,9 +261,9 @@ static inline void iterator_print_state(Iterator *self) { } #endif -unsigned ts_subtree_get_changed_ranges(Subtree *old_tree, Subtree *new_tree, - TSTreeCursor *cursor1, TSTreeCursor *cursor2, - const TSLanguage *language, TSRange **ranges) { +unsigned ts_subtree_get_changed_ranges(const Subtree *old_tree, const Subtree *new_tree, + TSTreeCursor *cursor1, TSTreeCursor *cursor2, + const TSLanguage *language, TSRange **ranges) { RangeArray results = array_new(); Iterator old_iter = iterator_new(cursor1, old_tree, language); diff --git a/src/runtime/get_changed_ranges.h b/src/runtime/get_changed_ranges.h index bf138a21..fbe9cc23 100644 --- a/src/runtime/get_changed_ranges.h +++ b/src/runtime/get_changed_ranges.h @@ -4,7 +4,8 @@ #include "runtime/subtree.h" unsigned ts_subtree_get_changed_ranges( - Subtree *old_tree, Subtree *new_tree, TSTreeCursor *cursor1, TSTreeCursor *cursor2, + const Subtree *old_tree, const Subtree *new_tree, + TSTreeCursor *cursor1, TSTreeCursor *cursor2, const TSLanguage *language, TSRange **ranges ); diff --git a/src/runtime/node.c b/src/runtime/node.c index e260f13b..1f221d12 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -47,7 +47,7 @@ static inline NodeChildIterator ts_node_child_iterator_begin(const TSNode *node) static inline bool ts_node_child_iterator_next(NodeChildIterator *self, TSNode *result) { if (self->child_index == self->parent->children.size) return false; - Subtree *child = self->parent->children.contents[self->child_index]; + const Subtree *child = self->parent->children.contents[self->child_index]; TSSymbol alias_symbol = 0; if (!child->extra) { if (self->alias_sequence) { diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 8332501b..dde46f80 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -41,8 +41,8 @@ static const unsigned MAX_SUMMARY_DEPTH = 16; static const unsigned MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE; typedef struct { - Subtree *token; - Subtree *last_external_token; + const Subtree *token; + const Subtree *last_external_token; uint32_t byte_index; } TokenCache; @@ -52,7 +52,7 @@ struct TSParser { SubtreePool tree_pool; const TSLanguage *language; ReduceActionSet reduce_actions; - Subtree *finished_tree; + const Subtree *finished_tree; Subtree scratch_tree; TokenCache token_cache; ReusableNode reusable_node; @@ -112,10 +112,10 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion versi for (uint32_t i = 0; i < pop.size; i++) { StackSlice slice = pop.contents[i]; TSStateId state = ts_stack_state(self->stack, slice.version); - Subtree *parent = *array_front(&slice.subtrees); + const Subtree *parent = *array_front(&slice.subtrees); for (uint32_t j = 0; j < parent->children.size; j++) { - Subtree *child = parent->children.contents[j]; + const Subtree *child = parent->children.contents[j]; pending = child->children.size > 0; if (child->symbol == ts_builtin_sym_error) { @@ -129,7 +129,7 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion versi } for (uint32_t j = 1; j < slice.subtrees.size; j++) { - Subtree *tree = slice.subtrees.contents[j]; + const Subtree *tree = slice.subtrees.contents[j]; ts_stack_push(self->stack, slice.version, tree, false, state); } @@ -144,11 +144,10 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion versi return did_break_down; } -static void ts_parser__breakdown_lookahead(TSParser *self, Subtree **lookahead, - TSStateId state, - ReusableNode *reusable_node) { +static void ts_parser__breakdown_lookahead(TSParser *self, const Subtree **lookahead, + TSStateId state, ReusableNode *reusable_node) { bool did_descend = false; - Subtree *tree = reusable_node_tree(reusable_node); + const Subtree *tree = reusable_node_tree(reusable_node); while (tree->children.size > 0 && tree->parse_state != state) { LOG("state_mismatch sym:%s", SYM_NAME(tree->symbol)); reusable_node_descend(reusable_node); @@ -243,7 +242,7 @@ static bool ts_parser__better_version_exists(TSParser *self, StackVersion versio return false; } -static void ts_parser__restore_external_scanner(TSParser *self, Subtree *external_token) { +static void ts_parser__restore_external_scanner(TSParser *self, const Subtree *external_token) { if (external_token) { self->language->external_scanner.deserialize( self->external_scanner_payload, @@ -255,9 +254,9 @@ static void ts_parser__restore_external_scanner(TSParser *self, Subtree *externa } } -static Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId parse_state) { +static const Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId parse_state) { Length start_position = ts_stack_position(self->stack, version); - Subtree *external_token = ts_stack_last_external_token(self->stack, version); + const Subtree *external_token = ts_stack_last_external_token(self->stack, version); TSLexMode lex_mode = self->language->lex_modes[parse_state]; const bool *valid_external_tokens = ts_language_enabled_external_tokens( self->language, @@ -403,8 +402,8 @@ static Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId p return result; } -static Subtree *ts_parser__get_cached_token(TSParser *self, size_t byte_index, - Subtree *last_external_token) { +static const Subtree *ts_parser__get_cached_token(TSParser *self, size_t byte_index, + const Subtree *last_external_token) { TokenCache *cache = &self->token_cache; if (cache->token && cache->byte_index == byte_index && @@ -415,8 +414,8 @@ static Subtree *ts_parser__get_cached_token(TSParser *self, size_t byte_index, } } -static void ts_parser__set_cached_token(TSParser *self, size_t byte_index, Subtree *last_external_token, - Subtree *token) { +static void ts_parser__set_cached_token(TSParser *self, size_t byte_index, + const Subtree *last_external_token, const Subtree *token) { TokenCache *cache = &self->token_cache; if (token) ts_subtree_retain(token); if (last_external_token) ts_subtree_retain(last_external_token); @@ -427,8 +426,8 @@ static void ts_parser__set_cached_token(TSParser *self, size_t byte_index, Subtr cache->last_external_token = last_external_token; } -static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, Subtree *tree, - TableEntry *table_entry) { +static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, const Subtree *tree, + TableEntry *table_entry) { TSLexMode current_lex_mode = self->language->lex_modes[state]; // If the token was created in a state with the same set of lookaheads, it is reusable. @@ -445,12 +444,13 @@ static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, Sub return current_lex_mode.external_lex_state == 0 && table_entry->is_reusable; } -static Subtree *ts_parser__get_lookahead(TSParser *self, StackVersion version, TSStateId *state, - ReusableNode *reusable_node, TableEntry *table_entry) { +static const Subtree *ts_parser__get_lookahead(TSParser *self, StackVersion version, + TSStateId *state, ReusableNode *reusable_node, + TableEntry *table_entry) { Length position = ts_stack_position(self->stack, version); - Subtree *last_external_token = ts_stack_last_external_token(self->stack, version); + const Subtree *last_external_token = ts_stack_last_external_token(self->stack, version); - Subtree *result; + const Subtree *result; while ((result = reusable_node_tree(reusable_node))) { uint32_t byte_offset = reusable_node_byte_offset(reusable_node); if (byte_offset > position.bytes) { @@ -523,7 +523,7 @@ static Subtree *ts_parser__get_lookahead(TSParser *self, StackVersion version, T return result; } -static bool ts_parser__select_tree(TSParser *self, Subtree *left, Subtree *right) { +static bool ts_parser__select_tree(TSParser *self, const Subtree *left, const Subtree *right) { if (!left) return true; if (!right) return false; @@ -574,23 +574,21 @@ static bool ts_parser__select_tree(TSParser *self, Subtree *left, Subtree *right } static void ts_parser__shift(TSParser *self, StackVersion version, TSStateId state, - Subtree *lookahead, bool extra) { + const Subtree *lookahead, bool extra) { + const Subtree *subtree_to_push; if (extra != lookahead->extra) { - if (ts_stack_version_count(self->stack) > 1) { - lookahead = ts_subtree_new_copy(&self->tree_pool, lookahead); - } else { - ts_subtree_retain(lookahead); - } - lookahead->extra = extra; + Subtree *result = ts_subtree_make_mut(&self->tree_pool, lookahead); + result->extra = extra; + subtree_to_push = result; } else { - ts_subtree_retain(lookahead); + subtree_to_push = lookahead; } - bool is_pending = lookahead->children.size > 0; - ts_stack_push(self->stack, version, lookahead, is_pending, state); - if (lookahead->has_external_tokens) { + bool is_pending = subtree_to_push->children.size > 0; + ts_stack_push(self->stack, version, subtree_to_push, is_pending, state); + if (subtree_to_push->has_external_tokens) { ts_stack_set_last_external_token( - self->stack, version, ts_subtree_last_external_token(lookahead) + self->stack, version, ts_subtree_last_external_token(subtree_to_push) ); } } @@ -717,19 +715,17 @@ static void ts_parser__start(TSParser *self, TSInput input, const Subtree *previ self->in_ambiguity = false; } -static void ts_parser__accept(TSParser *self, StackVersion version, Subtree *lookahead) { - lookahead->extra = true; +static void ts_parser__accept(TSParser *self, StackVersion version, const Subtree *lookahead) { assert(lookahead->symbol == ts_builtin_sym_end); - ts_subtree_retain(lookahead); ts_stack_push(self->stack, version, lookahead, false, 1); StackSliceArray pop = ts_stack_pop_all(self->stack, version); for (uint32_t i = 0; i < pop.size; i++) { SubtreeArray trees = pop.contents[i].subtrees; - Subtree *root = NULL; + const Subtree *root = NULL; for (uint32_t j = trees.size - 1; j + 1 > 0; j--) { - Subtree *child = trees.contents[j]; + const Subtree *child = trees.contents[j]; if (!child->extra) { for (uint32_t k = 0; k < child->children.size; k++) { ts_subtree_retain(child->children.contents[k]); @@ -875,7 +871,9 @@ static void ts_parser__handle_error(TSParser *self, StackVersion version, lookahead_symbol )) { StackVersion version_with_missing_tree = ts_stack_copy_version(self->stack, v); - Subtree *missing_tree = ts_subtree_new_missing_leaf(&self->tree_pool, missing_symbol, self->language); + const Subtree *missing_tree = ts_subtree_new_missing_leaf( + &self->tree_pool, missing_symbol, self->language + ); ts_stack_push( self->stack, version_with_missing_tree, missing_tree, false, @@ -930,11 +928,10 @@ static void ts_parser__halt_parse(TSParser *self) { Subtree *eof = ts_subtree_new_leaf(&self->tree_pool, ts_builtin_sym_end, length_zero(), length_zero(), self->language); ts_parser__accept(self, 0, eof); - ts_subtree_release(&self->tree_pool, eof); } static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, unsigned depth, - TSStateId goal_state) { + TSStateId goal_state) { StackSliceArray pop = ts_stack_pop_count(self->stack, version, depth); StackVersion previous_version = STACK_VERSION_NONE; @@ -975,7 +972,7 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, un } for (unsigned j = 0; j < trailing_extras.size; j++) { - Subtree *tree = trailing_extras.contents[j]; + const Subtree *tree = trailing_extras.contents[j]; ts_stack_push(self->stack, slice.version, tree, false, goal_state); } @@ -986,7 +983,7 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, un return previous_version != STACK_VERSION_NONE; } -static void ts_parser__recover(TSParser *self, StackVersion version, Subtree *lookahead) { +static void ts_parser__recover(TSParser *self, StackVersion version, const Subtree *lookahead) { bool did_recover = false; unsigned previous_version_count = ts_stack_version_count(self->stack); Length position = ts_stack_position(self->stack, version); @@ -1042,13 +1039,14 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree *lo if (did_recover && ts_stack_version_count(self->stack) > MAX_VERSION_COUNT) { ts_stack_halt(self->stack, version); + ts_subtree_release(&self->tree_pool, lookahead); return; } if (lookahead->symbol == ts_builtin_sym_end) { LOG("recover_eof"); SubtreeArray children = array_new(); - Subtree *parent = ts_subtree_new_error_node(&self->tree_pool, &children, self->language); + const Subtree *parent = ts_subtree_new_error_node(&self->tree_pool, &children, self->language); ts_stack_push(self->stack, version, parent, false, 1); ts_parser__accept(self, version, lookahead); return; @@ -1061,21 +1059,23 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree *lo if (ts_parser__better_version_exists(self, version, false, new_cost)) { ts_stack_halt(self->stack, version); + ts_subtree_release(&self->tree_pool, lookahead); return; } unsigned n; const TSParseAction *actions = ts_language_actions(self->language, 1, lookahead->symbol, &n); if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.extra) { - lookahead->extra = true; + Subtree *mutable_lookahead = ts_subtree_make_mut(&self->tree_pool, lookahead); + mutable_lookahead->extra = true; + lookahead = mutable_lookahead; } LOG("skip_token symbol:%s", SYM_NAME(lookahead->symbol)); - ts_subtree_retain(lookahead); SubtreeArray children = array_new(); array_reserve(&children, 1); array_push(&children, lookahead); - Subtree *error_repeat = ts_subtree_new_node( + const Subtree *error_repeat = ts_subtree_new_node( &self->tree_pool, ts_builtin_sym_error_repeat, &children, @@ -1110,7 +1110,9 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree *lo static void ts_parser__advance(TSParser *self, StackVersion version, ReusableNode *reusable_node) { TSStateId state = ts_stack_state(self->stack, version); TableEntry table_entry; - Subtree *lookahead = ts_parser__get_lookahead(self, version, &state, reusable_node, &table_entry); + const Subtree *lookahead = ts_parser__get_lookahead( + self, version, &state, reusable_node, &table_entry + ); for (;;) { StackVersion last_reduction_version = STACK_VERSION_NONE; @@ -1143,7 +1145,6 @@ static void ts_parser__advance(TSParser *self, StackVersion version, ReusableNod if (lookahead == reusable_node_tree(reusable_node)) { reusable_node_advance(reusable_node); } - ts_subtree_release(&self->tree_pool, lookahead); return; } @@ -1163,7 +1164,6 @@ static void ts_parser__advance(TSParser *self, StackVersion version, ReusableNod case TSParseActionTypeAccept: { LOG("accept"); ts_parser__accept(self, version, lookahead); - ts_subtree_release(&self->tree_pool, lookahead); return; } @@ -1175,7 +1175,6 @@ static void ts_parser__advance(TSParser *self, StackVersion version, ReusableNod if (lookahead == reusable_node_tree(reusable_node)) { reusable_node_advance(reusable_node); } - ts_subtree_release(&self->tree_pool, lookahead); return; } } @@ -1186,7 +1185,6 @@ static void ts_parser__advance(TSParser *self, StackVersion version, ReusableNod LOG_STACK(); } else if (state == ERROR_STATE) { ts_parser__recover(self, version, lookahead); - ts_subtree_release(&self->tree_pool, lookahead); return; } else if (!ts_parser__breakdown_top_of_stack(self, version)) { LOG("detect_error"); diff --git a/src/runtime/reusable_node.h b/src/runtime/reusable_node.h index c1d4f06b..42ae6f1e 100644 --- a/src/runtime/reusable_node.h +++ b/src/runtime/reusable_node.h @@ -1,21 +1,21 @@ #include "runtime/subtree.h" typedef struct { - Subtree *tree; + const Subtree *tree; uint32_t child_index; uint32_t byte_offset; } StackEntry; typedef struct { Array(StackEntry) stack; - Subtree *last_external_token; + const Subtree *last_external_token; } ReusableNode; static inline ReusableNode reusable_node_new() { return (ReusableNode) {array_new(), NULL}; } -static inline void reusable_node_reset(ReusableNode *self, Subtree *tree) { +static inline void reusable_node_reset(ReusableNode *self, const Subtree *tree) { array_clear(&self->stack); array_push(&self->stack, ((StackEntry) { .tree = tree, @@ -51,7 +51,7 @@ static inline void reusable_node_advance(ReusableNode *self) { self->last_external_token = ts_subtree_last_external_token(last_entry.tree); } - Subtree *tree; + const Subtree *tree; uint32_t next_index; do { StackEntry popped_entry = array_pop(&self->stack); diff --git a/src/runtime/stack.c b/src/runtime/stack.c index 21468e49..8921d559 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -21,7 +21,7 @@ typedef struct StackNode StackNode; typedef struct { StackNode *node; - Subtree *subtree; + const Subtree *subtree; bool is_pending; } StackLink; @@ -58,7 +58,7 @@ typedef enum { typedef struct { StackNode *node; - Subtree *last_external_token; + const Subtree *last_external_token; StackSummary *summary; unsigned node_count_at_last_error; TSSymbol lookahead_when_paused; @@ -119,8 +119,8 @@ recur: } } -static StackNode *stack_node_new(StackNode *previous_node, Subtree *subtree, bool is_pending, - TSStateId state, StackNodeArray *pool) { +static StackNode *stack_node_new(StackNode *previous_node, const Subtree *subtree, + bool is_pending, TSStateId state, StackNodeArray *pool) { StackNode *node = pool->size > 0 ? array_pop(pool) : ts_malloc(sizeof(StackNode)); @@ -380,11 +380,11 @@ Length ts_stack_position(const Stack *self, StackVersion version) { return array_get(&self->heads, version)->node->position; } -Subtree *ts_stack_last_external_token(const Stack *self, StackVersion version) { +const Subtree *ts_stack_last_external_token(const Stack *self, StackVersion version) { return array_get(&self->heads, version)->last_external_token; } -void ts_stack_set_last_external_token(Stack *self, StackVersion version, Subtree *token) { +void ts_stack_set_last_external_token(Stack *self, StackVersion version, const Subtree *token) { StackHead *head = array_get(&self->heads, version); if (token) ts_subtree_retain(token); if (head->last_external_token) ts_subtree_release(self->subtree_pool, head->last_external_token); @@ -410,7 +410,8 @@ unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version return head->node->node_count - head->node_count_at_last_error; } -void ts_stack_push(Stack *self, StackVersion version, Subtree *subtree, bool pending, TSStateId state) { +void ts_stack_push(Stack *self, StackVersion version, const Subtree *subtree, + bool pending, TSStateId state) { StackHead *head = array_get(&self->heads, version); StackNode *new_node = stack_node_new(head->node, subtree, pending, state, &self->node_pool); if (!subtree) head->node_count_at_last_error = new_node->node_count; @@ -684,7 +685,7 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) ); if (head->last_external_token) { - ExternalScannerState *state = &head->last_external_token->external_scanner_state; + const ExternalScannerState *state = &head->last_external_token->external_scanner_state; const char *data = ts_external_scanner_state_data(state); fprintf(f, "\nexternal_scanner_state:"); for (uint32_t j = 0; j < state->length; j++) fprintf(f, " %2X", data[j]); diff --git a/src/runtime/stack.h b/src/runtime/stack.h index 88c06bab..b7dba342 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -42,10 +42,10 @@ uint32_t ts_stack_version_count(const Stack *); TSStateId ts_stack_state(const Stack *, StackVersion); // Get the last external token associated with a given version of the stack. -Subtree *ts_stack_last_external_token(const Stack *, StackVersion); +const Subtree *ts_stack_last_external_token(const Stack *, StackVersion); // Set the last external token associated with a given version of the stack. -void ts_stack_set_last_external_token(Stack *, StackVersion, Subtree *); +void ts_stack_set_last_external_token(Stack *, StackVersion, const Subtree *); // Get the position of the given version of the stack within the document. Length ts_stack_position(const Stack *, StackVersion); @@ -55,7 +55,7 @@ Length ts_stack_position(const Stack *, StackVersion); // This transfers ownership of the tree to the Stack. Callers that // need to retain ownership of the tree for their own purposes should // first retain the tree. -void ts_stack_push(Stack *, StackVersion, Subtree *, bool, TSStateId); +void ts_stack_push(Stack *, StackVersion, const Subtree *, bool, TSStateId); // Pop the given number of entries from the given version of the stack. This // operation can increase the number of stack versions by revealing multiple diff --git a/src/runtime/subtree.c b/src/runtime/subtree.c index 0cd125bf..0991b67c 100644 --- a/src/runtime/subtree.c +++ b/src/runtime/subtree.c @@ -59,7 +59,7 @@ bool ts_external_scanner_state_eq(const ExternalScannerState *a, const ExternalS // SubtreeArray bool ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest) { - Subtree **contents = NULL; + const Subtree **contents = NULL; if (self.capacity > 0) { contents = ts_calloc(self.capacity, sizeof(Subtree *)); memcpy(contents, self.contents, self.size * sizeof(Subtree *)); @@ -86,7 +86,7 @@ SubtreeArray ts_subtree_array_remove_trailing_extras(SubtreeArray *self) { uint32_t i = self->size - 1; for (; i + 1 > 0; i--) { - Subtree *child = self->contents[i]; + const Subtree *child = self->contents[i]; if (!child->extra) break; array_push(&result, child); } @@ -99,7 +99,7 @@ SubtreeArray ts_subtree_array_remove_trailing_extras(SubtreeArray *self) { void ts_subtree_array_reverse(SubtreeArray *self) { for (uint32_t i = 0, limit = self->size / 2; i < limit; i++) { size_t reverse_index = self->size - 1 - i; - Subtree *swap = self->contents[i]; + const Subtree *swap = self->contents[i]; self->contents[i] = self->contents[reverse_index]; self->contents[reverse_index] = swap; } @@ -142,7 +142,7 @@ void ts_subtree_pool_free(SubtreePool *self, Subtree *tree) { // Subtree Subtree *ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, Length size, - const TSLanguage *language) { + const TSLanguage *language) { TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); Subtree *result = ts_subtree_pool_allocate(pool); *result = (Subtree){ @@ -163,11 +163,12 @@ Subtree *ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, }, .has_external_tokens = false, }; + if (symbol == ts_builtin_sym_end) result->extra = true; return result; } Subtree *ts_subtree_new_error(SubtreePool *pool, Length size, Length padding, - int32_t lookahead_char, const TSLanguage *language) { + int32_t lookahead_char, const TSLanguage *language) { Subtree *result = ts_subtree_new_leaf(pool, ts_builtin_sym_error, padding, size, language); result->fragile_left = true; result->fragile_right = true; @@ -175,7 +176,7 @@ Subtree *ts_subtree_new_error(SubtreePool *pool, Length size, Length padding, return result; } -Subtree *ts_subtree_new_copy(SubtreePool *pool, Subtree *self) { +Subtree *ts_subtree_new_copy(SubtreePool *pool, const Subtree *self) { Subtree *result = ts_subtree_pool_allocate(pool); *result = *self; if (result->children.size > 0) { @@ -185,9 +186,9 @@ Subtree *ts_subtree_new_copy(SubtreePool *pool, Subtree *self) { return result; } -static Subtree *ts_subtree_new_mut(SubtreePool *pool, Subtree *self) { +Subtree *ts_subtree_make_mut(SubtreePool *pool, const Subtree *self) { if (self->ref_count == 1) { - return self; + return (Subtree *)self; } else { Subtree *result = ts_subtree_new_copy(pool, self); ts_subtree_release(pool, self); @@ -196,21 +197,21 @@ static Subtree *ts_subtree_new_mut(SubtreePool *pool, Subtree *self) { } static void ts_subtree__compress(Subtree *self, unsigned count, const TSLanguage *language, - SubtreeArray *stack) { + MutableSubtreeArray *stack) { unsigned initial_stack_size = stack->size; Subtree *tree = self; for (unsigned i = 0; i < count; i++) { if (tree->ref_count > 1 || tree->children.size != 2) break; - Subtree *child = tree->children.contents[0]; + Subtree *child = (Subtree *)tree->children.contents[0]; if ( child->ref_count > 1 || child->children.size != 2 || child->symbol != tree->symbol ) break; - Subtree *grandchild = child->children.contents[0]; + Subtree *grandchild = (Subtree *)child->children.contents[0]; if ( grandchild->ref_count > 1 || grandchild->children.size != 2 || @@ -227,38 +228,41 @@ static void ts_subtree__compress(Subtree *self, unsigned count, const TSLanguage while (stack->size > initial_stack_size) { tree = array_pop(stack); assert(tree); - Subtree *child = tree->children.contents[0]; - Subtree *grandchild = child->children.contents[1]; + Subtree *child = (Subtree *)tree->children.contents[0]; + Subtree *grandchild = (Subtree *)child->children.contents[1]; ts_subtree_set_children(grandchild, &grandchild->children, language); ts_subtree_set_children(child, &child->children, language); ts_subtree_set_children(tree, &tree->children, language); } } -void ts_subtree_balance(Subtree *self, SubtreePool *pool, const TSLanguage *language) { +void ts_subtree_balance(const Subtree *self, SubtreePool *pool, const TSLanguage *language) { array_clear(&pool->tree_stack); - array_push(&pool->tree_stack, self); + + if (self->ref_count == 1) { + array_push(&pool->tree_stack, (Subtree *)self); + } + while (pool->tree_stack.size > 0) { Subtree *tree = array_pop(&pool->tree_stack); assert(tree); - if (tree->repeat_depth > 0) { - if (tree->children.contents[0]->repeat_depth > tree->children.contents[1]->repeat_depth) { - unsigned n = ( - tree->children.contents[0]->repeat_depth - - tree->children.contents[1]->repeat_depth - ); - for (unsigned i = n / 2; i > 0; i /= 2) { - ts_subtree__compress(tree, i, language, &pool->tree_stack); - n -= i; - } + if (tree->repeat_depth > 0 && + tree->children.contents[0]->repeat_depth > tree->children.contents[1]->repeat_depth) { + unsigned n = ( + tree->children.contents[0]->repeat_depth - + tree->children.contents[1]->repeat_depth + ); + for (unsigned i = n / 2; i > 0; i /= 2) { + ts_subtree__compress(tree, i, language, &pool->tree_stack); + n -= i; } } for (uint32_t i = 0; i < tree->children.size; i++) { - Subtree *child = tree->children.contents[i]; + const Subtree *child = tree->children.contents[i]; if (child->ref_count == 1) { - array_push(&pool->tree_stack, child); + array_push(&pool->tree_stack, (Subtree *)child); } } } @@ -282,7 +286,7 @@ void ts_subtree_set_children(Subtree *self, SubtreeArray *children, const TSLang const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); for (uint32_t i = 0; i < self->children.size; i++) { - Subtree *child = self->children.contents[i]; + const Subtree *child = self->children.contents[i]; if (i == 0) { self->padding = child->padding; @@ -328,7 +332,7 @@ void ts_subtree_set_children(Subtree *self, SubtreeArray *children, const TSLang ERROR_COST_PER_SKIPPED_CHAR * self->size.bytes + ERROR_COST_PER_SKIPPED_LINE * self->size.extent.row; for (uint32_t i = 0; i < self->children.size; i++) { - Subtree *child = self->children.contents[i]; + const Subtree *child = self->children.contents[i]; if (child->extra) continue; if (child->symbol == ts_builtin_sym_error && child->children.size == 0) continue; if (child->visible) { @@ -340,8 +344,8 @@ void ts_subtree_set_children(Subtree *self, SubtreeArray *children, const TSLang } if (self->children.size > 0) { - Subtree *first_child = self->children.contents[0]; - Subtree *last_child = self->children.contents[self->children.size - 1]; + const Subtree *first_child = self->children.contents[0]; + const Subtree *last_child = self->children.contents[self->children.size - 1]; self->first_leaf = first_child->first_leaf; if (first_child->fragile_left) self->fragile_left = true; if (last_child->fragile_right) self->fragile_right = true; @@ -361,7 +365,7 @@ void ts_subtree_set_children(Subtree *self, SubtreeArray *children, const TSLang } Subtree *ts_subtree_new_node(SubtreePool *pool, TSSymbol symbol, SubtreeArray *children, - unsigned alias_sequence_id, const TSLanguage *language) { + unsigned alias_sequence_id, const TSLanguage *language) { Subtree *result = ts_subtree_new_leaf(pool, symbol, length_zero(), length_zero(), language); result->alias_sequence_id = alias_sequence_id; if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { @@ -373,41 +377,46 @@ Subtree *ts_subtree_new_node(SubtreePool *pool, TSSymbol symbol, SubtreeArray *c } Subtree *ts_subtree_new_error_node(SubtreePool *pool, SubtreeArray *children, - const TSLanguage *language) { + const TSLanguage *language) { return ts_subtree_new_node(pool, ts_builtin_sym_error, children, 0, language); } Subtree *ts_subtree_new_missing_leaf(SubtreePool *pool, TSSymbol symbol, - const TSLanguage *language) { + const TSLanguage *language) { Subtree *result = ts_subtree_new_leaf(pool, symbol, length_zero(), length_zero(), language); result->is_missing = true; result->error_cost = ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY; return result; } -void ts_subtree_retain(Subtree *self) { +void ts_subtree_retain(const Subtree *self) { assert(self->ref_count > 0); - atomic_inc(&self->ref_count); + atomic_inc((volatile uint32_t *)&self->ref_count); assert(self->ref_count != 0); } -void ts_subtree_release(SubtreePool *pool, Subtree *self) { +void ts_subtree_release(SubtreePool *pool, const Subtree *self) { array_clear(&pool->tree_stack); - array_push(&pool->tree_stack, self); + + assert(self->ref_count > 0); + if (atomic_dec((volatile uint32_t *)&self->ref_count) == 0) { + array_push(&pool->tree_stack, (Subtree *)self); + } + while (pool->tree_stack.size > 0) { Subtree *tree = array_pop(&pool->tree_stack); - assert(tree->ref_count > 0); - if (atomic_dec(&tree->ref_count) == 0) { - if (tree->children.size > 0) { - for (uint32_t i = 0; i < tree->children.size; i++) { - array_push(&pool->tree_stack, tree->children.contents[i]); + if (tree->children.size > 0) { + for (uint32_t i = 0; i < tree->children.size; i++) { + const Subtree *child = tree->children.contents[i]; + if (atomic_dec((volatile uint32_t *)&child->ref_count) == 0) { + array_push(&pool->tree_stack, (Subtree *)child); } - array_delete(&tree->children); - } else if (tree->has_external_tokens) { - ts_external_scanner_state_delete(&tree->external_scanner_state); } - ts_subtree_pool_free(pool, tree); + array_delete(&tree->children); + } else if (tree->has_external_tokens) { + ts_external_scanner_state_delete(&tree->external_scanner_state); } + ts_subtree_pool_free(pool, tree); } } @@ -446,8 +455,8 @@ int ts_subtree_compare(const Subtree *left, const Subtree *right) { if (right->children.size < left->children.size) return 1; for (uint32_t i = 0; i < left->children.size; i++) { - Subtree *left_child = left->children.contents[i]; - Subtree *right_child = right->children.contents[i]; + const Subtree *left_child = left->children.contents[i]; + const Subtree *right_child = right->children.contents[i]; switch (ts_subtree_compare(left_child, right_child)) { case -1: return -1; @@ -460,17 +469,17 @@ int ts_subtree_compare(const Subtree *left, const Subtree *right) { return 0; } -Subtree *ts_subtree_invalidate_lookahead(Subtree *self, uint32_t edit_byte_offset, - SubtreePool *pool) { +const Subtree *ts_subtree_invalidate_lookahead(const Subtree *self, uint32_t edit_byte_offset, + SubtreePool *pool) { if (edit_byte_offset >= self->bytes_scanned) return self; - Subtree *result = ts_subtree_new_mut(pool, self); + Subtree *result = ts_subtree_make_mut(pool, self); result->has_changes = true; if (result->children.size > 0) { uint32_t child_start_byte = 0; for (uint32_t i = 0; i < result->children.size; i++) { - Subtree **child = &result->children.contents[i]; + const Subtree **child = &result->children.contents[i]; if (child_start_byte > edit_byte_offset) break; *child = ts_subtree_invalidate_lookahead(*child, edit_byte_offset - child_start_byte, pool); child_start_byte += ts_subtree_total_bytes(*child); @@ -480,11 +489,11 @@ Subtree *ts_subtree_invalidate_lookahead(Subtree *self, uint32_t edit_byte_offse return result; } -Subtree *ts_subtree__edit(Subtree *self, Edit edit, SubtreePool *pool) { +const Subtree *ts_subtree__edit(const Subtree *self, Edit edit, SubtreePool *pool) { Length new_end = length_add(edit.start, edit.added); Length old_end = length_add(edit.start, edit.removed); - Subtree *result = ts_subtree_new_mut(pool, self); + Subtree *result = ts_subtree_make_mut(pool, self); result->has_changes = true; if (old_end.bytes <= result->padding.bytes) { @@ -501,7 +510,7 @@ Subtree *ts_subtree__edit(Subtree *self, Edit edit, SubtreePool *pool) { Length child_left, child_right = length_zero(); for (uint32_t i = 0; i < result->children.size; i++) { - Subtree **child = &result->children.contents[i]; + const Subtree **child = &result->children.contents[i]; Length child_size = ts_subtree_total_size(*child); child_left = child_right; child_right = length_add(child_left, child_size); @@ -537,7 +546,7 @@ Subtree *ts_subtree__edit(Subtree *self, Edit edit, SubtreePool *pool) { return result; } -Subtree *ts_subtree_edit(Subtree *self, const TSInputEdit *edit, SubtreePool *pool) { +const Subtree *ts_subtree_edit(const Subtree *self, const TSInputEdit *edit, SubtreePool *pool) { return ts_subtree__edit(self, (Edit) { .start = {edit->start_byte, edit->start_point}, .added = {edit->bytes_added, edit->extent_added}, @@ -545,11 +554,11 @@ Subtree *ts_subtree_edit(Subtree *self, const TSInputEdit *edit, SubtreePool *po }, pool); } -Subtree *ts_subtree_last_external_token(Subtree *tree) { +const Subtree *ts_subtree_last_external_token(const Subtree *tree) { if (!tree->has_external_tokens) return NULL; while (tree->children.size > 0) { for (uint32_t i = tree->children.size - 1; i + 1 > 0; i--) { - Subtree *child = tree->children.contents[i]; + const Subtree *child = tree->children.contents[i]; if (child->has_external_tokens) { tree = child; break; @@ -611,7 +620,7 @@ static size_t ts_subtree__write_to_string(const Subtree *self, char *string, siz const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); uint32_t structural_child_index = 0; for (uint32_t i = 0; i < self->children.size; i++) { - Subtree *child = self->children.contents[i]; + const Subtree *child = self->children.contents[i]; if (child->extra) { cursor += ts_subtree__write_to_string( child, *writer, limit, diff --git a/src/runtime/subtree.h b/src/runtime/subtree.h index da69383d..cf59db43 100644 --- a/src/runtime/subtree.h +++ b/src/runtime/subtree.h @@ -24,7 +24,8 @@ typedef struct { typedef struct Subtree Subtree; -typedef Array(Subtree *) SubtreeArray; +typedef Array(const Subtree *) SubtreeArray; +typedef Array(Subtree *) MutableSubtreeArray; struct Subtree { Length padding; @@ -71,8 +72,8 @@ struct Subtree { }; typedef struct { - SubtreeArray free_trees; - SubtreeArray tree_stack; + MutableSubtreeArray free_trees; + MutableSubtreeArray tree_stack; } SubtreePool; void ts_external_scanner_state_init(ExternalScannerState *, const char *, unsigned); @@ -90,20 +91,21 @@ void ts_subtree_pool_free(SubtreePool *, Subtree *); Subtree *ts_subtree_new_leaf(SubtreePool *, TSSymbol, Length, Length, const TSLanguage *); Subtree *ts_subtree_new_node(SubtreePool *, TSSymbol, SubtreeArray *, unsigned, const TSLanguage *); -Subtree *ts_subtree_new_copy(SubtreePool *, Subtree *child); +Subtree *ts_subtree_new_copy(SubtreePool *, const Subtree *); Subtree *ts_subtree_new_error_node(SubtreePool *, SubtreeArray *, const TSLanguage *); Subtree *ts_subtree_new_error(SubtreePool *, Length, Length, int32_t, const TSLanguage *); Subtree *ts_subtree_new_missing_leaf(SubtreePool *, TSSymbol, const TSLanguage *); -void ts_subtree_retain(Subtree *tree); -void ts_subtree_release(SubtreePool *, Subtree *tree); +Subtree *ts_subtree_make_mut(SubtreePool *, const Subtree *); +void ts_subtree_retain(const Subtree *tree); +void ts_subtree_release(SubtreePool *, const Subtree *tree); bool ts_subtree_eq(const Subtree *tree1, const Subtree *tree2); int ts_subtree_compare(const Subtree *tree1, const Subtree *tree2); void ts_subtree_set_children(Subtree *, SubtreeArray *, const TSLanguage *); -void ts_subtree_balance(Subtree *, SubtreePool *, const TSLanguage *); -Subtree *ts_subtree_edit(Subtree *, const TSInputEdit *edit, SubtreePool *); +void ts_subtree_balance(const Subtree *, SubtreePool *, const TSLanguage *); +const Subtree *ts_subtree_edit(const Subtree *, const TSInputEdit *edit, SubtreePool *); char *ts_subtree_string(const Subtree *, const TSLanguage *, bool include_all); void ts_subtree_print_dot_graph(const Subtree *, const TSLanguage *, FILE *); -Subtree *ts_subtree_last_external_token(Subtree *); +const Subtree *ts_subtree_last_external_token(const Subtree *); bool ts_subtree_external_scanner_state_eq(const Subtree *, const Subtree *); static inline uint32_t ts_subtree_total_bytes(const Subtree *self) { diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 2c365644..3e8a145e 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -17,7 +17,7 @@ TSTree *ts_tree_copy(const TSTree *self) { return ts_tree_new(self->root, self->language); } -void ts_tree_delete(const TSTree *self) { +void ts_tree_delete(TSTree *self) { SubtreePool pool = ts_subtree_pool_new(0); ts_subtree_release(&pool, self->root); ts_subtree_pool_delete(&pool); diff --git a/src/runtime/tree_cursor.c b/src/runtime/tree_cursor.c index c7aa9691..31eee8b4 100644 --- a/src/runtime/tree_cursor.c +++ b/src/runtime/tree_cursor.c @@ -28,7 +28,7 @@ void ts_tree_cursor_delete(TSTreeCursor *self) { bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { TreeCursorEntry *last_entry = array_back(&self->stack); - Subtree *tree = last_entry->subtree; + const Subtree *tree = last_entry->subtree; Length position = last_entry->position; bool did_descend; @@ -37,7 +37,7 @@ bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { uint32_t structural_child_index = 0; for (uint32_t i = 0; i < tree->children.size; i++) { - Subtree *child = tree->children.contents[i]; + const Subtree *child = tree->children.contents[i]; if (child->visible || child->visible_child_count > 0) { array_push(&self->stack, ((TreeCursorEntry) { .subtree = child, @@ -68,11 +68,11 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) { TreeCursorEntry *parent_entry = &self->stack.contents[i]; - Subtree *parent = parent_entry->subtree; + const Subtree *parent = parent_entry->subtree; uint32_t child_index = child_entry->child_index; uint32_t structural_child_index = child_entry->structural_child_index; Length position = child_entry->position; - Subtree *child = parent->children.contents[child_index]; + const Subtree *child = parent->children.contents[child_index]; while (++child_index < parent->children.size) { if (!child->extra) structural_child_index++; diff --git a/src/runtime/tree_cursor.h b/src/runtime/tree_cursor.h index c016b276..615e7e06 100644 --- a/src/runtime/tree_cursor.h +++ b/src/runtime/tree_cursor.h @@ -4,7 +4,7 @@ #include "runtime/subtree.h" typedef struct { - Subtree *subtree; + const Subtree *subtree; Length position; uint32_t child_index; uint32_t structural_child_index; diff --git a/test/helpers/tree_helpers.cc b/test/helpers/tree_helpers.cc index 7fd015a3..0e63a19f 100644 --- a/test/helpers/tree_helpers.cc +++ b/test/helpers/tree_helpers.cc @@ -14,11 +14,11 @@ const char *symbol_names[24] = { "twenty-two", "twenty-three" }; -SubtreeArray *tree_array(std::vector trees) { +SubtreeArray *tree_array(std::vector trees) { static SubtreeArray result; result.capacity = trees.size(); result.size = trees.size(); - result.contents = (Subtree **)calloc(trees.size(), sizeof(Subtree *)); + result.contents = (const Subtree **)calloc(trees.size(), sizeof(Subtree *)); for (size_t i = 0; i < trees.size(); i++) { result.contents[i] = trees[i]; } @@ -49,7 +49,7 @@ bool operator==(const TSNode &left, const TSNode &right) { return ts_node_eq(left, right); } -bool operator==(const std::vector &vec, const SubtreeArray &array) { +bool operator==(const std::vector &vec, const SubtreeArray &array) { if (vec.size() != array.size) return false; for (size_t i = 0; i < array.size; i++) diff --git a/test/helpers/tree_helpers.h b/test/helpers/tree_helpers.h index bd08aad7..00740faf 100644 --- a/test/helpers/tree_helpers.h +++ b/test/helpers/tree_helpers.h @@ -6,12 +6,12 @@ #include extern const char *symbol_names[24]; -SubtreeArray *tree_array(std::vector trees); +SubtreeArray *tree_array(std::vector trees); std::ostream &operator<<(std::ostream &stream, const Subtree *tree); std::ostream &operator<<(std::ostream &stream, const TSNode &node); bool operator==(const TSNode &left, const TSNode &right); -bool operator==(const std::vector &right, const SubtreeArray &array); +bool operator==(const std::vector &right, const SubtreeArray &array); void assert_consistent_tree_sizes(TSNode node); diff --git a/test/runtime/stack_test.cc b/test/runtime/stack_test.cc index 67812a07..68104935 100644 --- a/test/runtime/stack_test.cc +++ b/test/runtime/stack_test.cc @@ -44,6 +44,10 @@ void free_slice_array(SubtreePool *pool, StackSliceArray *slices) { } } +Subtree *mutate(const Subtree *subtree) { + return (Subtree *)subtree; +} + struct StackEntry { TSStateId state; size_t depth; @@ -69,7 +73,7 @@ START_TEST describe("Stack", [&]() { Stack *stack; const size_t subtree_count = 11; - Subtree *subtrees[subtree_count]; + const Subtree *subtrees[subtree_count]; Length tree_len = {3, {0, 3}}; SubtreePool pool; @@ -84,7 +88,7 @@ describe("Stack", [&]() { dummy_language.symbol_metadata = symbol_metadata; for (size_t i = 0; i < subtree_count; i++) { - subtrees[i] = ts_subtree_new_leaf(&pool, i, length_zero(), tree_len, &dummy_language); + subtrees[i] = ts_subtree_new_leaf(&pool, i + 1, length_zero(), tree_len, &dummy_language); } }); @@ -99,7 +103,7 @@ describe("Stack", [&]() { AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); }); - auto push = [&](StackVersion version, Subtree *tree, TSStateId state) { + auto push = [&](StackVersion version, const Subtree *tree, TSStateId state) { ts_subtree_retain(tree); ts_stack_push(stack, version, tree, false, state); }; @@ -181,7 +185,7 @@ describe("Stack", [&]() { // . <──0── A <──1── B <────3──── D* // ↑ // └───2─── C <──4── D* - subtrees[3]->size = tree_len * 3; + mutate(subtrees[3])->size = tree_len * 3; push(0, subtrees[1], stateB); push(1, subtrees[2], stateC); push(0, subtrees[3], stateD); @@ -224,8 +228,8 @@ describe("Stack", [&]() { // . <──0── A <────1──── B* // ↑ // └2─ A <──1── B* - subtrees[2]->extra = true; - subtrees[2]->size = tree_len * 0; + mutate(subtrees[2])->extra = true; + mutate(subtrees[2])->size = tree_len * 0; push(0, subtrees[1], stateB); push(1, subtrees[2], stateA); @@ -261,14 +265,14 @@ describe("Stack", [&]() { StackSlice slice = pop.contents[0]; AssertThat(slice.version, Equals(1)); - AssertThat(slice.subtrees, Equals(vector({ subtrees[1], subtrees[2] }))); + AssertThat(slice.subtrees, Equals(vector({ subtrees[1], subtrees[2] }))); AssertThat(ts_stack_state(stack, 1), Equals(stateA)); free_slice_array(&pool,&pop); }); it("does not count 'extra' subtrees toward the given count", [&]() { - subtrees[1]->extra = true; + mutate(subtrees[1])->extra = true; // . <──0── A <──1── B <──2── C* // ↑ @@ -277,7 +281,7 @@ describe("Stack", [&]() { AssertThat(pop.size, Equals(1)); StackSlice slice = pop.contents[0]; - AssertThat(slice.subtrees, Equals(vector({ subtrees[0], subtrees[1], subtrees[2] }))); + AssertThat(slice.subtrees, Equals(vector({ subtrees[0], subtrees[1], subtrees[2] }))); AssertThat(ts_stack_state(stack, 1), Equals(1)); free_slice_array(&pool,&pop); @@ -322,11 +326,11 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.subtrees, Equals(vector({ subtrees[2], subtrees[3], subtrees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[2], subtrees[3], subtrees[10] }))); StackSlice slice2 = pop.contents[1]; AssertThat(slice2.version, Equals(2)); - AssertThat(slice2.subtrees, Equals(vector({ subtrees[5], subtrees[6], subtrees[10] }))); + AssertThat(slice2.subtrees, Equals(vector({ subtrees[5], subtrees[6], subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(3)); AssertThat(get_stack_entries(stack, 0), Equals(vector({ @@ -366,7 +370,7 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.subtrees, Equals(vector({ subtrees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(2)); AssertThat(ts_stack_state(stack, 0), Equals(stateI)); @@ -388,11 +392,11 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.subtrees, Equals(vector({ subtrees[1], subtrees[2], subtrees[3], subtrees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[1], subtrees[2], subtrees[3], subtrees[10] }))); StackSlice slice2 = pop.contents[1]; AssertThat(slice2.version, Equals(1)); - AssertThat(slice2.subtrees, Equals(vector({ subtrees[4], subtrees[5], subtrees[6], subtrees[10] }))); + AssertThat(slice2.subtrees, Equals(vector({ subtrees[4], subtrees[5], subtrees[6], subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(2)); AssertThat(ts_stack_state(stack, 0), Equals(stateI)); @@ -443,15 +447,15 @@ describe("Stack", [&]() { StackSlice slice1 = pop.contents[0]; AssertThat(slice1.version, Equals(1)); - AssertThat(slice1.subtrees, Equals(vector({ subtrees[3], subtrees[10] }))); + AssertThat(slice1.subtrees, Equals(vector({ subtrees[3], subtrees[10] }))); StackSlice slice2 = pop.contents[1]; AssertThat(slice2.version, Equals(2)); - AssertThat(slice2.subtrees, Equals(vector({ subtrees[6], subtrees[10] }))); + AssertThat(slice2.subtrees, Equals(vector({ subtrees[6], subtrees[10] }))); StackSlice slice3 = pop.contents[2]; AssertThat(slice3.version, Equals(3)); - AssertThat(slice3.subtrees, Equals(vector({ subtrees[9], subtrees[10] }))); + AssertThat(slice3.subtrees, Equals(vector({ subtrees[9], subtrees[10] }))); AssertThat(ts_stack_version_count(stack), Equals(4)); AssertThat(ts_stack_state(stack, 0), Equals(stateI)); @@ -489,8 +493,8 @@ describe("Stack", [&]() { ts_stack_push(stack, 0, subtrees[1], true, stateB); ts_subtree_retain(subtrees[1]); - subtrees[2]->extra = true; - subtrees[3]->extra = true; + mutate(subtrees[2])->extra = true; + mutate(subtrees[3])->extra = true; push(0, subtrees[2], stateB); push(0, subtrees[3], stateB); @@ -498,7 +502,7 @@ describe("Stack", [&]() { StackSliceArray pop = ts_stack_pop_pending(stack, 0); AssertThat(pop.size, Equals(1)); - AssertThat(pop.contents[0].subtrees, Equals(vector({ subtrees[1], subtrees[2], subtrees[3] }))); + AssertThat(pop.contents[0].subtrees, Equals(vector({ subtrees[1], subtrees[2], subtrees[3] }))); AssertThat(get_stack_entries(stack, 0), Equals(vector({ {stateA, 0}, @@ -526,10 +530,10 @@ describe("Stack", [&]() { describe("setting external token state", [&]() { before_each([&]() { - subtrees[1]->has_external_tokens = true; - subtrees[2]->has_external_tokens = true; - ts_external_scanner_state_init(&subtrees[1]->external_scanner_state, NULL, 0); - ts_external_scanner_state_init(&subtrees[2]->external_scanner_state, NULL, 0); + mutate(subtrees[1])->has_external_tokens = true; + mutate(subtrees[2])->has_external_tokens = true; + ts_external_scanner_state_init(&mutate(subtrees[1])->external_scanner_state, NULL, 0); + ts_external_scanner_state_init(&mutate(subtrees[2])->external_scanner_state, NULL, 0); }); it("allows the state to be retrieved", [&]() { @@ -546,8 +550,8 @@ describe("Stack", [&]() { }); it("does not merge stack versions with different external token states", [&]() { - ts_external_scanner_state_init(&subtrees[1]->external_scanner_state, "abcd", 2); - ts_external_scanner_state_init(&subtrees[2]->external_scanner_state, "ABCD", 2); + ts_external_scanner_state_init(&mutate(subtrees[1])->external_scanner_state, "abcd", 2); + ts_external_scanner_state_init(&mutate(subtrees[2])->external_scanner_state, "ABCD", 2); ts_stack_copy_version(stack, 0); push(0, subtrees[0], 5); @@ -560,8 +564,8 @@ describe("Stack", [&]() { }); it("merges stack versions with identical external token states", [&]() { - ts_external_scanner_state_init(&subtrees[1]->external_scanner_state, "abcd", 2); - ts_external_scanner_state_init(&subtrees[2]->external_scanner_state, "abcd", 2); + ts_external_scanner_state_init(&mutate(subtrees[1])->external_scanner_state, "abcd", 2); + ts_external_scanner_state_init(&mutate(subtrees[2])->external_scanner_state, "abcd", 2); ts_stack_copy_version(stack, 0); push(0, subtrees[0], 5); diff --git a/test/runtime/subtree_test.cc b/test/runtime/subtree_test.cc index 4d32ec32..3c1c9ad2 100644 --- a/test/runtime/subtree_test.cc +++ b/test/runtime/subtree_test.cc @@ -10,7 +10,7 @@ void assert_consistent(const Subtree *tree) { Length total_children_size = length_zero(); for (size_t i = 0; i < tree->children.size; i++) { - Subtree *child = tree->children.contents[i]; + const Subtree *child = tree->children.contents[i]; assert_consistent(child); total_children_size = length_add(total_children_size, ts_subtree_total_size(child)); } @@ -50,7 +50,7 @@ describe("Subtree", []() { describe("make_leaf", [&]() { it("does not mark the tree as fragile", [&]() { - Subtree *tree = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); + const Subtree *tree = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); AssertThat(tree->fragile_left, IsFalse()); AssertThat(tree->fragile_right, IsFalse()); @@ -60,7 +60,7 @@ describe("Subtree", []() { describe("make_error", [&]() { it("marks the tree as fragile", [&]() { - Subtree *error_tree = ts_subtree_new_error( + const Subtree *error_tree = ts_subtree_new_error( &pool, length_zero(), length_zero(), @@ -76,7 +76,7 @@ describe("Subtree", []() { }); describe("make_node", [&]() { - Subtree *tree1, *tree2, *parent1; + const Subtree *tree1, *tree2, *parent1; before_each([&]() { tree1 = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); @@ -104,11 +104,12 @@ describe("Subtree", []() { }); describe("when the first node is fragile on the left side", [&]() { - Subtree *parent; + const Subtree *parent; before_each([&]() { - tree1->fragile_left = true; - tree1->extra = true; + Subtree *mutable_tree1 = (Subtree *)tree1; + mutable_tree1->fragile_left = true; + mutable_tree1->extra = true; ts_subtree_retain(tree1); ts_subtree_retain(tree2); @@ -128,11 +129,12 @@ describe("Subtree", []() { }); describe("when the last node is fragile on the right side", [&]() { - Subtree *parent; + const Subtree *parent; before_each([&]() { - tree2->fragile_right = true; - tree2->extra = true; + Subtree *mutable_tree2 = (Subtree *)tree2; + mutable_tree2->fragile_right = true; + mutable_tree2->extra = true; ts_subtree_retain(tree1); ts_subtree_retain(tree2); @@ -152,11 +154,13 @@ describe("Subtree", []() { }); describe("when the outer nodes aren't fragile on their outer side", [&]() { - Subtree *parent; + const Subtree *parent; before_each([&]() { - tree1->fragile_right = true; - tree2->fragile_left = true; + Subtree *mutable_tree1 = (Subtree *)tree1; + Subtree *mutable_tree2 = (Subtree *)tree2; + mutable_tree1->fragile_right = true; + mutable_tree2->fragile_left = true; ts_subtree_retain(tree1); ts_subtree_retain(tree2); @@ -178,7 +182,7 @@ describe("Subtree", []() { }); describe("edit", [&]() { - Subtree *tree; + const Subtree *tree; before_each([&]() { tree = ts_subtree_new_node(&pool, symbol1, tree_array({ @@ -205,7 +209,7 @@ describe("Subtree", []() { edit.extent_added = {0, 1}; ts_subtree_retain(tree); - Subtree *new_tree = ts_subtree_edit(tree, &edit, &pool); + const Subtree *new_tree = ts_subtree_edit(tree, &edit, &pool); assert_consistent(tree); assert_consistent(new_tree); @@ -352,7 +356,8 @@ describe("Subtree", []() { describe("edits within a tree's range of scanned bytes", [&]() { it("marks preceding trees as changed", [&]() { - tree->children.contents[0]->bytes_scanned = 7; + Subtree *mutable_child = (Subtree *)tree->children.contents[0]; + mutable_child->bytes_scanned = 7; TSInputEdit edit; edit.start_byte = 6; @@ -370,7 +375,7 @@ describe("Subtree", []() { }); describe("eq", [&]() { - Subtree *leaf; + const Subtree *leaf; before_each([&]() { leaf = ts_subtree_new_leaf(&pool, symbol1, {2, {0, 1}}, {5, {0, 4}}, &language); @@ -381,17 +386,17 @@ describe("Subtree", []() { }); it("returns true for identical trees", [&]() { - Subtree *leaf_copy = ts_subtree_new_leaf(&pool, symbol1, {2, {1, 1}}, {5, {1, 4}}, &language); + const Subtree *leaf_copy = ts_subtree_new_leaf(&pool, symbol1, {2, {1, 1}}, {5, {1, 4}}, &language); AssertThat(ts_subtree_eq(leaf, leaf_copy), IsTrue()); - Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({ + const Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({ leaf, leaf_copy, }), 0, &language); ts_subtree_retain(leaf); ts_subtree_retain(leaf_copy); - Subtree *parent_copy = ts_subtree_new_node(&pool, symbol2, tree_array({ + const Subtree *parent_copy = ts_subtree_new_node(&pool, symbol2, tree_array({ leaf, leaf_copy, }), 0, &language); @@ -406,7 +411,7 @@ describe("Subtree", []() { }); it("returns false for trees with different symbols", [&]() { - Subtree *different_leaf = ts_subtree_new_leaf( + const Subtree *different_leaf = ts_subtree_new_leaf( &pool, leaf->symbol + 1, leaf->padding, @@ -419,14 +424,16 @@ describe("Subtree", []() { }); it("returns false for trees with different options", [&]() { - Subtree *different_leaf = ts_subtree_new_leaf(&pool, leaf->symbol, leaf->padding, leaf->size, &language); - different_leaf->visible = !leaf->visible; + const Subtree *different_leaf = ts_subtree_new_leaf( + &pool, leaf->symbol, leaf->padding, leaf->size, &language + ); + ((Subtree *)different_leaf)->visible = !leaf->visible; AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); ts_subtree_release(&pool, different_leaf); }); it("returns false for trees with different paddings or sizes", [&]() { - Subtree *different_leaf = ts_subtree_new_leaf(&pool, leaf->symbol, {}, leaf->size, &language); + const Subtree *different_leaf = ts_subtree_new_leaf(&pool, leaf->symbol, {}, leaf->size, &language); AssertThat(ts_subtree_eq(leaf, different_leaf), IsFalse()); ts_subtree_release(&pool, different_leaf); @@ -436,16 +443,16 @@ describe("Subtree", []() { }); it("returns false for trees with different children", [&]() { - Subtree *leaf2 = ts_subtree_new_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); + const Subtree *leaf2 = ts_subtree_new_leaf(&pool, symbol2, {1, {0, 1}}, {3, {0, 3}}, &language); - Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({ + const Subtree *parent = ts_subtree_new_node(&pool, symbol2, tree_array({ leaf, leaf2, }), 0, &language); ts_subtree_retain(leaf); ts_subtree_retain(leaf2); - Subtree *different_parent = ts_subtree_new_node(&pool, symbol2, tree_array({ + const Subtree *different_parent = ts_subtree_new_node(&pool, symbol2, tree_array({ leaf2, leaf, }), 0, &language); @@ -465,13 +472,13 @@ describe("Subtree", []() { Length padding = {1, {0, 1}}; Length size = {2, {0, 2}}; - auto make_external = [](Subtree *tree) { - tree->has_external_tokens = true; + auto make_external = [](const Subtree *tree) { + ((Subtree *)tree)->has_external_tokens = true; return tree; }; it("returns the last serialized external token state in the given tree", [&]() { - Subtree *tree1, *tree2, *tree3, *tree4, *tree5, *tree6, *tree7, *tree8, *tree9; + const Subtree *tree1, *tree2, *tree3, *tree4, *tree5, *tree6, *tree7, *tree8, *tree9; tree1 = ts_subtree_new_node(&pool, symbol1, tree_array({ (tree2 = ts_subtree_new_node(&pool, symbol2, tree_array({ From a3e08e7c315d35b0936673f650b64406263b9a10 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 16:10:36 -0700 Subject: [PATCH 28/31] Add randomized multi-threaded tests on parse trees Co-Authored-By: Rick Winfrey --- test/helpers/random_helpers.cc | 37 +++++++++----------- test/helpers/random_helpers.h | 23 ++++++++++--- test/integration/real_grammars.cc | 6 ++-- test/runtime/tree_test.cc | 57 +++++++++++++++++++++++++++++++ test/tests.cc | 2 +- 5 files changed, 96 insertions(+), 29 deletions(-) diff --git a/test/helpers/random_helpers.cc b/test/helpers/random_helpers.cc index d79475c7..b3de98e2 100644 --- a/test/helpers/random_helpers.cc +++ b/test/helpers/random_helpers.cc @@ -1,3 +1,4 @@ +#include "helpers/random_helpers.h" #include #include #include @@ -6,54 +7,50 @@ using std::string; using std::vector; -static std::default_random_engine engine; +Generator default_generator(0); unsigned get_time_as_seed() { return time(nullptr); } -void random_reseed(unsigned seed) { +void Generator::reseed(unsigned seed) { engine.seed(seed); } -unsigned random_unsigned() { - return std::uniform_int_distribution()(engine); +unsigned Generator::operator()() { + return distribution(engine); } -unsigned random_unsigned(unsigned max) { - return std::uniform_int_distribution(0, max - 1)(engine); +unsigned Generator::operator()(unsigned max) { + return distribution(engine) % max; } -static string random_string(char min, char max) { +string Generator::str(char min, char max) { string result; - size_t length = random_unsigned(12); + size_t length = operator()(12); for (size_t i = 0; i < length; i++) { - result += (min + random_unsigned(max - min)); + result += (min + operator()(max - min)); } return result; } -static string random_char(string characters) { - size_t index = random_unsigned(characters.size()); - return string() + characters[index]; -} - -string random_words(size_t count) { +string Generator::words(size_t count) { string result; bool just_inserted_word = false; for (size_t i = 0; i < count; i++) { - if (random_unsigned(10) < 6) { - result += random_char("!(){}[]<>+-="); + if (operator()(10) < 6) { + const char *operator_characters = "!(){}[]<>+-="; + result += operator_characters[operator()(strlen(operator_characters))]; } else { if (just_inserted_word) result += " "; - result += random_string('a', 'z'); + result += str('a', 'z'); just_inserted_word = true; } } return result; } -string select_random(const vector &list) { - return list[random_unsigned(list.size())]; +string Generator::select(const vector &list) { + return list[operator()(list.size())]; } diff --git a/test/helpers/random_helpers.h b/test/helpers/random_helpers.h index b66c4aee..d415aafe 100644 --- a/test/helpers/random_helpers.h +++ b/test/helpers/random_helpers.h @@ -3,12 +3,25 @@ #include #include +#include unsigned get_time_as_seed(); -void random_reseed(unsigned); -unsigned random_unsigned(); -unsigned random_unsigned(unsigned max); -std::string random_words(size_t count); -std::string select_random(const std::vector &); + +class Generator { + std::default_random_engine engine; + std::uniform_int_distribution distribution; + +public: + Generator(uint32_t seed) : engine{seed} {} + + void reseed(unsigned); + unsigned operator()(); + unsigned operator()(unsigned max); + std::string words(size_t count); + std::string str(char min, char max); + std::string select(const std::vector &); +}; + +extern Generator default_generator; #endif // HELPERS_RANDOM_HELPERS_H_ diff --git a/test/integration/real_grammars.cc b/test/integration/real_grammars.cc index a628177f..02752a34 100644 --- a/test/integration/real_grammars.cc +++ b/test/integration/real_grammars.cc @@ -76,9 +76,9 @@ for (auto &language_name : test_languages) { set> insertions; for (size_t i = 0; i < 60; i++) { - size_t edit_position = random_unsigned(utf8_char_count(entry.input)); - size_t deletion_size = random_unsigned(utf8_char_count(entry.input) - edit_position); - string inserted_text = random_words(random_unsigned(4) + 1); + size_t edit_position = default_generator(utf8_char_count(entry.input)); + size_t deletion_size = default_generator(utf8_char_count(entry.input) - edit_position); + string inserted_text = default_generator.words(default_generator(4) + 1); if (insertions.insert({edit_position, inserted_text}).second) { it(("parses " + entry.description + diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc index a87c754b..6d038220 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/tree_test.cc @@ -1,4 +1,5 @@ #include "test_helper.h" +#include #include "runtime/alloc.h" #include "helpers/record_alloc.h" #include "helpers/stream_methods.h" @@ -8,6 +9,11 @@ #include "helpers/stderr_logger.h" #include "helpers/spy_input.h" #include "helpers/load_language.h" +#include "helpers/random_helpers.h" +#include "helpers/read_test_entries.h" +#include "helpers/encoding_helpers.h" +#include "helpers/tree_helpers.h" +#include TSPoint point(uint32_t row, uint32_t column) { TSPoint result = {row, column}; @@ -37,6 +43,57 @@ describe("Tree", [&]() { AssertThat(actual, Equals(expected)); }; + describe("copy()", [&]() { + it("returns a tree that can be safely used while the current tree is edited", [&]() { + const TSLanguage *language = load_real_language("javascript"); + ts_parser_set_language(parser, language); + string source_code = examples_for_language("javascript")[0].input; + + input = new SpyInput(source_code, 32); + TSTree *original_tree = ts_parser_parse(parser, nullptr, input->input()); + + vector> new_trees; + for (unsigned i = 0; i < 8; i++) { + TSTree *tree_copy = ts_tree_copy(original_tree); + new_trees.push_back(std::async([i, tree_copy, &source_code, language]() { + Generator random(TREE_SITTER_SEED + i); + + TSTree *tree = tree_copy; + TSParser *parser = ts_parser_new(); + ts_parser_set_language(parser, language); + SpyInput *input = new SpyInput(source_code, 32); + + for (unsigned j = 0; j < 30; j++) { + usleep(random(200)); + + size_t edit_position = random(utf8_char_count(input->content)); + size_t deletion_size = random(utf8_char_count(input->content) - edit_position); + string inserted_text = random.words(random(4) + 1); + + TSInputEdit edit = input->replace(edit_position, deletion_size, inserted_text); + ts_tree_edit(tree, &edit); + + TSTree *new_tree = ts_parser_parse(parser, tree, input->input()); + ts_tree_delete(tree); + tree = new_tree; + } + + ts_parser_delete(parser); + delete input; + + return tree; + })); + } + + for (auto &future : new_trees) { + future.wait(); + TSTree *new_tree = future.get(); + assert_consistent_tree_sizes(ts_tree_root_node(new_tree)); + ts_tree_delete(new_tree); + } + }); + }); + describe("get_changed_ranges()", [&]() { before_each([&]() { ts_parser_set_language(parser, load_real_language("javascript")); diff --git a/test/tests.cc b/test/tests.cc index 303f9059..0d8c23e7 100644 --- a/test/tests.cc +++ b/test/tests.cc @@ -12,7 +12,7 @@ int main(int argc, char *argv[]) { } printf("Random seed: %d\n", TREE_SITTER_SEED); - random_reseed(TREE_SITTER_SEED); + default_generator.reseed(TREE_SITTER_SEED); return bandit::run(argc, argv); } From 043a2fc0d99defac87b21a696b842ad8350dca0e Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 16:53:47 -0700 Subject: [PATCH 29/31] Assert absence of memory leaks in randomized multi-threaded tree test --- src/runtime/tree.c | 2 +- test/helpers/record_alloc.cc | 16 ++++++++++++++-- test/helpers/record_alloc.h | 2 +- test/runtime/tree_test.cc | 13 ++++++++----- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 3e8a145e..ce1e2b98 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -37,7 +37,7 @@ TSNode ts_tree_root_node(const TSTree *self) { void ts_tree_edit(TSTree *self, const TSInputEdit *edit) { SubtreePool pool = ts_subtree_pool_new(0); self->root = ts_subtree_edit(self->root, edit, &pool); - assert(pool.tree_stack.capacity == 0 && pool.free_trees.capacity == 0); + ts_subtree_pool_delete(&pool); } TSRange *ts_tree_get_changed_ranges(const TSTree *self, const TSTree *other, uint32_t *count) { diff --git a/test/helpers/record_alloc.cc b/test/helpers/record_alloc.cc index 43e11abe..2e2ea648 100644 --- a/test/helpers/record_alloc.cc +++ b/test/helpers/record_alloc.cc @@ -1,6 +1,7 @@ #include #include #include +#include using std::map; using std::vector; @@ -8,13 +9,16 @@ using std::vector; static bool _enabled = false; static size_t _allocation_count = 0; static map _outstanding_allocations; +static std::mutex _outstanding_allocations_mutex; +static bool _multi_threaded_mode = false; namespace record_alloc { -void start() { +void start(bool multi_threaded_mode) { _enabled = true; _allocation_count = 0; _outstanding_allocations.clear(); + _multi_threaded_mode = multi_threaded_mode; } void stop() { @@ -30,7 +34,11 @@ vector outstanding_allocation_indices() { } size_t allocation_count() { - return _allocation_count; + size_t result; + _outstanding_allocations_mutex.lock(); + result = _allocation_count; + _outstanding_allocations_mutex.unlock(); + return result; } } // namespace record_alloc @@ -39,16 +47,20 @@ extern "C" { static void *record_allocation(void *result) { if (!_enabled) return result; + if (_multi_threaded_mode) _outstanding_allocations_mutex.lock(); _outstanding_allocations[result] = _allocation_count; _allocation_count++; + if (_multi_threaded_mode) _outstanding_allocations_mutex.unlock(); return result; } static void record_deallocation(void *pointer) { + if (_multi_threaded_mode) _outstanding_allocations_mutex.lock(); auto entry = _outstanding_allocations.find(pointer); if (entry != _outstanding_allocations.end()) { _outstanding_allocations.erase(entry); } + if (_multi_threaded_mode) _outstanding_allocations_mutex.unlock(); } void *ts_record_malloc(size_t size) { diff --git a/test/helpers/record_alloc.h b/test/helpers/record_alloc.h index 1f5968ac..f21876b4 100644 --- a/test/helpers/record_alloc.h +++ b/test/helpers/record_alloc.h @@ -5,7 +5,7 @@ namespace record_alloc { -void start(); +void start(bool multi_threaded_mode = false); void stop(); void fail_at_allocation_index(size_t failure_index); std::vector outstanding_allocation_indices(); diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc index 6d038220..ad9d3204 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/tree_test.cc @@ -28,11 +28,17 @@ describe("Tree", [&]() { TSTree *tree; before_each([&]() { + record_alloc::start(true); parser = ts_parser_new(); + tree = nullptr; + input = nullptr; }); after_each([&]() { + if (tree) ts_tree_delete(tree); + if (input) delete input; ts_parser_delete(parser); + AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); }); auto assert_root_node = [&](const string &expected) { @@ -85,6 +91,8 @@ describe("Tree", [&]() { })); } + ts_tree_delete(original_tree); + for (auto &future : new_trees) { future.wait(); TSTree *new_tree = future.get(); @@ -105,11 +113,6 @@ describe("Tree", [&]() { ); }); - after_each([&]() { - ts_tree_delete(tree); - delete input; - }); - auto get_changed_ranges_for_edit = [&](function fn) -> vector { TSInputEdit edit = fn(); ts_tree_edit(tree, &edit); From 32c06b9b5988849eb2041cb51771c235ef48d3d9 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 17:08:46 -0700 Subject: [PATCH 30/31] Make multi-threaded test work on windows Co-Authored-By: Rick Winfrey --- test/helpers/random_helpers.cc | 23 +++++++++++++++++++++-- test/helpers/random_helpers.h | 1 + test/runtime/tree_test.cc | 7 +++---- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/test/helpers/random_helpers.cc b/test/helpers/random_helpers.cc index b3de98e2..b0200e51 100644 --- a/test/helpers/random_helpers.cc +++ b/test/helpers/random_helpers.cc @@ -34,13 +34,14 @@ string Generator::str(char min, char max) { return result; } +static string operator_characters = "!(){}[]<>+-="; + string Generator::words(size_t count) { string result; bool just_inserted_word = false; for (size_t i = 0; i < count; i++) { if (operator()(10) < 6) { - const char *operator_characters = "!(){}[]<>+-="; - result += operator_characters[operator()(strlen(operator_characters))]; + result += operator_characters[operator()(operator_characters.size())]; } else { if (just_inserted_word) result += " "; @@ -54,3 +55,21 @@ string Generator::words(size_t count) { string Generator::select(const vector &list) { return list[operator()(list.size())]; } + +#ifdef _WIN32 + +#include + +void Generator::sleep_some() { + Sleep(operator()(5)); +} + +#else + +#include + +void Generator::sleep_some() { + usleep(operator()(5 * 1000)); +} + +#endif diff --git a/test/helpers/random_helpers.h b/test/helpers/random_helpers.h index d415aafe..7dd471fd 100644 --- a/test/helpers/random_helpers.h +++ b/test/helpers/random_helpers.h @@ -20,6 +20,7 @@ public: std::string words(size_t count); std::string str(char min, char max); std::string select(const std::vector &); + void sleep_some(); }; extern Generator default_generator; diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc index ad9d3204..b599f568 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/tree_test.cc @@ -13,7 +13,6 @@ #include "helpers/read_test_entries.h" #include "helpers/encoding_helpers.h" #include "helpers/tree_helpers.h" -#include TSPoint point(uint32_t row, uint32_t column) { TSPoint result = {row, column}; @@ -67,10 +66,10 @@ describe("Tree", [&]() { TSTree *tree = tree_copy; TSParser *parser = ts_parser_new(); ts_parser_set_language(parser, language); - SpyInput *input = new SpyInput(source_code, 32); + SpyInput *input = new SpyInput(source_code, 1024); - for (unsigned j = 0; j < 30; j++) { - usleep(random(200)); + for (unsigned j = 0; j < 10; j++) { + random.sleep_some(); size_t edit_position = random(utf8_char_count(input->content)); size_t deletion_size = random(utf8_char_count(input->content) - edit_position); From b2c5741deddbe5b428242f367f90395faa503234 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 May 2018 17:19:16 -0700 Subject: [PATCH 31/31] Link tests against libpthread on linux --- tests.gyp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests.gyp b/tests.gyp index ab0de485..80e8d618 100644 --- a/tests.gyp +++ b/tests.gyp @@ -101,7 +101,7 @@ 'cflags_cc': ['-std=c++14'], 'conditions': [ ['OS=="linux"', { - 'libraries': ['-ldl'], + 'libraries': ['-ldl', '-lpthread'], }] ], 'xcode_settings': {