diff --git a/src/runtime/node.c b/src/runtime/node.c index 35267964..8cf3b662 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -33,7 +33,11 @@ static inline uint32_t ts_node__offset_row(TSNode self) { static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { const Tree *tree = ts_node__tree(self); - return include_anonymous ? tree->visible : tree->visible && tree->named; + if (tree->context.rename_symbol > 0) { + return true; + } else { + return include_anonymous ? tree->visible : tree->visible && tree->named; + } } static inline uint32_t ts_node__relevant_child_count(TSNode self, diff --git a/src/runtime/tree.c b/src/runtime/tree.c index ddbab2cd..8e7c0801 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -159,18 +159,20 @@ void ts_tree_assign_parents(Tree *self, TreePath *path, const TSLanguage *langua Tree *tree = array_pop(path).tree; Length offset = length_zero(); const TSSymbol *rename_sequence = ts_language_rename_sequence(language, tree->rename_sequence_id); + uint32_t non_extra_index = 0; for (uint32_t i = 0; i < tree->child_count; i++) { Tree *child = tree->children[i]; if (child->context.parent != tree || child->context.index != i) { child->context.parent = tree; child->context.index = i; child->context.offset = offset; - if (rename_sequence && rename_sequence[i] != 0) { - child->context.rename_symbol = rename_sequence[i]; + if (!child->extra && rename_sequence && rename_sequence[non_extra_index] != 0) { + child->context.rename_symbol = rename_sequence[non_extra_index]; } array_push(path, ((TreePathEntry){child, length_zero(), 0})); } offset = length_add(offset, ts_tree_total_size(child)); + if (!child->extra) non_extra_index++; } } } @@ -189,6 +191,8 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children, self->has_external_tokens = false; self->dynamic_precedence = 0; + uint32_t non_extra_index = 0; + for (uint32_t i = 0; i < child_count; i++) { Tree *child = children[i]; @@ -207,7 +211,7 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children, if (child->visible) { self->visible_child_count++; - if (child->named || (rename_sequence && rename_sequence[i] != 0)) { + if (child->named || (!child->extra && rename_sequence && rename_sequence[non_extra_index] != 0)) { self->named_child_count++; } } else if (child->child_count > 0) { @@ -221,6 +225,8 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children, self->fragile_left = self->fragile_right = true; self->parse_state = TS_TREE_STATE_NONE; } + + if (!child->extra) non_extra_index++; } if (self->symbol == ts_builtin_sym_error) { diff --git a/test/runtime/node_test.cc b/test/runtime/node_test.cc index 00c6a267..3326b4d2 100644 --- a/test/runtime/node_test.cc +++ b/test/runtime/node_test.cc @@ -8,50 +8,70 @@ START_TEST -describe("Node", []() { - TSDocument *document; - TSNode array_node; - string input_string = - "\n" - "\n" - "[\n" - " 123,\n" - " false,\n" - " {\n" - " \"x\": null\n" - " }\n" - "]"; +string json_string = R"JSON( - size_t array_index = input_string.find("[\n"); - size_t array_end_index = input_string.find("]") + 1; - size_t number_index = input_string.find("123"); - size_t number_end_index = number_index + string("123").size(); - size_t false_index = input_string.find("false"); - size_t false_end_index = false_index + string("false").size(); - size_t object_index = input_string.find("{"); - size_t object_end_index = input_string.find("}") + 1; - size_t string_index = input_string.find("\"x\""); - size_t string_end_index = string_index + 3; - size_t colon_index = input_string.find(":"); - size_t null_index = input_string.find("null"); - size_t null_end_index = null_index + string("null").size(); +[ + 123, + false, + { + "x": null + } +] +)JSON"; + +size_t array_index = json_string.find("[\n"); +size_t array_end_index = json_string.find("]") + 1; +size_t number_index = json_string.find("123"); +size_t number_end_index = number_index + string("123").size(); +size_t false_index = json_string.find("false"); +size_t false_end_index = false_index + string("false").size(); +size_t object_index = json_string.find("{"); +size_t object_end_index = json_string.find("}") + 1; +size_t string_index = json_string.find("\"x\""); +size_t string_end_index = string_index + 3; +size_t colon_index = json_string.find(":"); +size_t null_index = json_string.find("null"); +size_t null_end_index = null_index + string("null").size(); + +string grammar_with_renames_and_extras = R"JSON({ + "name": "renames_and_extras", + + "extras": [ + {"type": "PATTERN", "value": "\\s+"}, + {"type": "SYMBOL", "name": "comment"}, + ], + + "rules": { + "a": { + "type": "SEQ", + "members": [ + {"type": "SYMBOL", "name": "b"}, + { + "type": "RENAME", + "value": "B", + "content": {"type": "SYMBOL", "name": "b"} + }, + {"type": "SYMBOL", "name": "b"} + ] + }, + + "b": {"type": "STRING", "value": "b"}, + "comment": {"type": "STRING", "value": "..."} + } +})JSON"; + +describe("Node", [&]() { + TSDocument *document; + TSNode root_node; before_each([&]() { record_alloc::start(); document = ts_document_new(); ts_document_set_language(document, load_real_language("json")); - ts_document_set_input_string(document, input_string.c_str()); + ts_document_set_input_string(document, json_string.c_str()); ts_document_parse(document); - - array_node = ts_document_root_node(document); - char *node_string = ts_node_string(array_node, document); - AssertThat(node_string, Equals( - "(array " - "(number) " - "(false) " - "(object (pair (string) (null))))")); - ts_free(node_string); + root_node = ts_document_root_node(document); }); after_each([&]() { @@ -61,21 +81,31 @@ describe("Node", []() { AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); }); + it("parses the example as expected (precondition)", [&]() { + char *node_string = ts_node_string(root_node, document); + AssertThat(node_string, Equals( + "(array " + "(number) " + "(false) " + "(object (pair (string) (null))))")); + ts_free(node_string); + }); + describe("named_child_count(), named_child(i)", [&]() { it("returns the named child node at the given index", [&]() { - AssertThat(ts_node_type(array_node, document), Equals("array")); + AssertThat(ts_node_type(root_node, document), Equals("array")); - AssertThat(ts_node_named_child_count(array_node), Equals(3)); - AssertThat(ts_node_start_byte(array_node), Equals(array_index)); - AssertThat(ts_node_end_byte(array_node), Equals(array_end_index)); - AssertThat(ts_node_start_char(array_node), Equals(array_index)); - AssertThat(ts_node_end_char(array_node), Equals(array_end_index)); - AssertThat(ts_node_start_point(array_node), Equals({ 2, 0 })); - AssertThat(ts_node_end_point(array_node), Equals({ 8, 1 })); + AssertThat(ts_node_named_child_count(root_node), Equals(3)); + AssertThat(ts_node_start_byte(root_node), Equals(array_index)); + AssertThat(ts_node_end_byte(root_node), Equals(array_end_index)); + AssertThat(ts_node_start_char(root_node), Equals(array_index)); + AssertThat(ts_node_end_char(root_node), Equals(array_end_index)); + AssertThat(ts_node_start_point(root_node), Equals({ 2, 0 })); + AssertThat(ts_node_end_point(root_node), Equals({ 8, 1 })); - TSNode number_node = ts_node_named_child(array_node, 0); - TSNode false_node = ts_node_named_child(array_node, 1); - TSNode object_node = ts_node_named_child(array_node, 2); + TSNode number_node = ts_node_named_child(root_node, 0); + TSNode false_node = ts_node_named_child(root_node, 1); + TSNode object_node = ts_node_named_child(root_node, 2); AssertThat(ts_node_type(number_node, document), Equals("number")); AssertThat(ts_node_type(false_node, document), Equals("false")); @@ -127,10 +157,32 @@ describe("Node", []() { AssertThat(ts_node_parent(string_node), Equals(pair_node)); AssertThat(ts_node_parent(null_node), Equals(pair_node)); AssertThat(ts_node_parent(pair_node), Equals(object_node)); - AssertThat(ts_node_parent(number_node), Equals(array_node)); - AssertThat(ts_node_parent(false_node), Equals(array_node)); - AssertThat(ts_node_parent(object_node), Equals(array_node)); - AssertThat(ts_node_parent(array_node).data, Equals(nullptr)); + AssertThat(ts_node_parent(number_node), Equals(root_node)); + AssertThat(ts_node_parent(false_node), Equals(root_node)); + AssertThat(ts_node_parent(object_node), Equals(root_node)); + AssertThat(ts_node_parent(root_node).data, Equals(nullptr)); + }); + + it("works correctly when the node contains renamed children and extras", [&]() { + TSCompileResult compile_result = ts_compile_grammar(grammar_with_renames_and_extras.c_str()); + const TSLanguage *language = load_test_language("renames_and_extras", compile_result); + ts_document_set_language(document, language); + ts_document_set_input_string(document, "b ... b ... b"); + ts_document_parse(document); + root_node = ts_document_root_node(document); + + char *node_string = ts_node_string(root_node, document); + AssertThat(node_string, Equals("(a (b) (comment) (B) (comment) (b))")); + ts_free(node_string); + + AssertThat(ts_node_named_child_count(root_node), Equals(5u)); + AssertThat(ts_node_type(ts_node_named_child(root_node, 0), document), Equals("b")); + AssertThat(ts_node_type(ts_node_named_child(root_node, 1), document), Equals("comment")); + AssertThat(ts_node_type(ts_node_named_child(root_node, 2), document), Equals("B")); + AssertThat(ts_node_type(ts_node_named_child(root_node, 3), document), Equals("comment")); + AssertThat(ts_node_type(ts_node_named_child(root_node, 4), document), Equals("b")); + + AssertThat(ts_node_symbol(ts_node_named_child(root_node, 0)), !Equals(ts_node_symbol(ts_node_named_child(root_node, 2)))); }); }); @@ -138,7 +190,7 @@ describe("Node", []() { it("returns an iterator that yields each of the node's symbols", [&]() { const TSLanguage *language = ts_document_language(document); - TSNode false_node = ts_node_descendant_for_char_range(array_node, false_index, false_index + 1); + TSNode false_node = ts_node_descendant_for_char_range(root_node, false_index, false_index + 1); TSSymbolIterator iterator = ts_node_symbols(false_node); AssertThat(iterator.done, Equals(false)); AssertThat(ts_language_symbol_name(language, iterator.value), Equals("false")); @@ -150,7 +202,7 @@ describe("Node", []() { ts_symbol_iterator_next(&iterator); AssertThat(iterator.done, Equals(true)); - TSNode comma_node = ts_node_descendant_for_char_range(array_node, number_end_index, number_end_index); + TSNode comma_node = ts_node_descendant_for_char_range(root_node, number_end_index, number_end_index); iterator = ts_node_symbols(comma_node); AssertThat(iterator.done, Equals(false)); AssertThat(ts_language_symbol_name(language, iterator.value), Equals(",")); @@ -162,16 +214,16 @@ describe("Node", []() { describe("child_count(), child(i)", [&]() { it("returns the child node at the given index, including anonymous nodes", [&]() { - AssertThat(ts_node_child_count(array_node), Equals(7)); - TSNode child1 = ts_node_child(array_node, 0); - TSNode child2 = ts_node_child(array_node, 1); - TSNode child3 = ts_node_child(array_node, 2); - TSNode child4 = ts_node_child(array_node, 3); - TSNode child5 = ts_node_child(array_node, 4); - TSNode child6 = ts_node_child(array_node, 5); - TSNode child7 = ts_node_child(array_node, 6); + AssertThat(ts_node_child_count(root_node), Equals(7)); + TSNode child1 = ts_node_child(root_node, 0); + TSNode child2 = ts_node_child(root_node, 1); + TSNode child3 = ts_node_child(root_node, 2); + TSNode child4 = ts_node_child(root_node, 3); + TSNode child5 = ts_node_child(root_node, 4); + TSNode child6 = ts_node_child(root_node, 5); + TSNode child7 = ts_node_child(root_node, 6); - AssertThat(ts_node_type(array_node, document), Equals("array")); + AssertThat(ts_node_type(root_node, document), Equals("array")); AssertThat(ts_node_type(child1, document), Equals("[")); AssertThat(ts_node_type(child2, document), Equals("number")); AssertThat(ts_node_type(child3, document), Equals(",")); @@ -180,7 +232,7 @@ describe("Node", []() { AssertThat(ts_node_type(child6, document), Equals("object")); AssertThat(ts_node_type(child7, document), Equals("]")); - AssertThat(ts_node_is_named(array_node), IsTrue()); + AssertThat(ts_node_is_named(root_node), IsTrue()); AssertThat(ts_node_is_named(child1), IsFalse()); AssertThat(ts_node_is_named(child2), IsTrue()); AssertThat(ts_node_is_named(child3), IsFalse()); @@ -233,32 +285,32 @@ describe("Node", []() { AssertThat(ts_node_parent(left_brace), Equals(child6)); AssertThat(ts_node_parent(pair), Equals(child6)); AssertThat(ts_node_parent(right_brace), Equals(child6)); - AssertThat(ts_node_parent(child1), Equals(array_node)); - AssertThat(ts_node_parent(child2), Equals(array_node)); - AssertThat(ts_node_parent(child3), Equals(array_node)); - AssertThat(ts_node_parent(child4), Equals(array_node)); - AssertThat(ts_node_parent(child5), Equals(array_node)); - AssertThat(ts_node_parent(child6), Equals(array_node)); - AssertThat(ts_node_parent(child7), Equals(array_node)); - AssertThat(ts_node_parent(array_node).data, Equals(nullptr)); + AssertThat(ts_node_parent(child1), Equals(root_node)); + AssertThat(ts_node_parent(child2), Equals(root_node)); + AssertThat(ts_node_parent(child3), Equals(root_node)); + AssertThat(ts_node_parent(child4), Equals(root_node)); + AssertThat(ts_node_parent(child5), Equals(root_node)); + AssertThat(ts_node_parent(child6), Equals(root_node)); + AssertThat(ts_node_parent(child7), Equals(root_node)); + AssertThat(ts_node_parent(root_node).data, Equals(nullptr)); }); }); describe("next_sibling(), prev_sibling()", [&]() { it("returns the node's next and previous sibling, including anonymous nodes", [&]() { - TSNode bracket_node1 = ts_node_child(array_node, 0); - TSNode number_node = ts_node_child(array_node, 1); - TSNode array_comma_node1 = ts_node_child(array_node, 2); - TSNode false_node = ts_node_child(array_node, 3); - TSNode array_comma_node2 = ts_node_child(array_node, 4); - TSNode object_node = ts_node_child(array_node, 5); + TSNode bracket_node1 = ts_node_child(root_node, 0); + TSNode number_node = ts_node_child(root_node, 1); + TSNode array_comma_node1 = ts_node_child(root_node, 2); + TSNode false_node = ts_node_child(root_node, 3); + TSNode array_comma_node2 = ts_node_child(root_node, 4); + TSNode object_node = ts_node_child(root_node, 5); TSNode brace_node1 = ts_node_child(object_node, 0); TSNode pair_node = ts_node_child(object_node, 1); TSNode string_node = ts_node_child(pair_node, 0); TSNode colon_node = ts_node_child(pair_node, 1); TSNode null_node = ts_node_child(pair_node, 2); TSNode brace_node2 = ts_node_child(object_node, 2); - TSNode bracket_node2 = ts_node_child(array_node, 6); + TSNode bracket_node2 = ts_node_child(root_node, 6); AssertThat(ts_node_next_sibling(bracket_node1), Equals(number_node)); AssertThat(ts_node_next_sibling(number_node), Equals(array_comma_node1)); @@ -294,16 +346,16 @@ describe("Node", []() { }); it("returns null when the node has no parent", [&]() { - AssertThat(ts_node_next_named_sibling(array_node).data, Equals(nullptr)); - AssertThat(ts_node_prev_named_sibling(array_node).data, Equals(nullptr)); + AssertThat(ts_node_next_named_sibling(root_node).data, Equals(nullptr)); + AssertThat(ts_node_prev_named_sibling(root_node).data, Equals(nullptr)); }); }); describe("next_named_sibling(), prev_named_sibling()", [&]() { it("returns the node's next and previous siblings", [&]() { - TSNode number_node = ts_node_named_child(array_node, 0); - TSNode false_node = ts_node_named_child(array_node, 1); - TSNode object_node = ts_node_named_child(array_node, 2); + TSNode number_node = ts_node_named_child(root_node, 0); + TSNode false_node = ts_node_named_child(root_node, 1); + TSNode object_node = ts_node_named_child(root_node, 2); TSNode pair_node = ts_node_named_child(object_node, 0); TSNode string_node = ts_node_named_child(pair_node, 0); TSNode null_node = ts_node_named_child(pair_node, 1); @@ -317,22 +369,22 @@ describe("Node", []() { }); it("returns null when the node has no parent", [&]() { - AssertThat(ts_node_next_named_sibling(array_node).data, Equals(nullptr)); - AssertThat(ts_node_prev_named_sibling(array_node).data, Equals(nullptr)); + AssertThat(ts_node_next_named_sibling(root_node).data, Equals(nullptr)); + AssertThat(ts_node_prev_named_sibling(root_node).data, Equals(nullptr)); }); }); describe("named_descendant_for_char_range(start, end)", [&]() { describe("when there is a leaf node that spans the given range exactly", [&]() { it("returns that leaf node", [&]() { - TSNode leaf = ts_node_named_descendant_for_char_range(array_node, string_index, string_end_index - 1); + TSNode leaf = ts_node_named_descendant_for_char_range(root_node, string_index, string_end_index - 1); AssertThat(ts_node_type(leaf, document), Equals("string")); AssertThat(ts_node_start_byte(leaf), Equals(string_index)); AssertThat(ts_node_end_byte(leaf), Equals(string_end_index)); AssertThat(ts_node_start_point(leaf), Equals({ 6, 4 })); AssertThat(ts_node_end_point(leaf), Equals({ 6, 7 })); - leaf = ts_node_named_descendant_for_char_range(array_node, number_index, number_end_index - 1); + leaf = ts_node_named_descendant_for_char_range(root_node, number_index, number_end_index - 1); AssertThat(ts_node_type(leaf, document), Equals("number")); AssertThat(ts_node_start_byte(leaf), Equals(number_index)); AssertThat(ts_node_end_byte(leaf), Equals(number_end_index)); @@ -343,14 +395,14 @@ describe("Node", []() { describe("when there is a leaf node that extends beyond the given range", [&]() { it("returns that leaf node", [&]() { - TSNode leaf = ts_node_named_descendant_for_char_range(array_node, string_index, string_index + 1); + TSNode leaf = ts_node_named_descendant_for_char_range(root_node, string_index, string_index + 1); AssertThat(ts_node_type(leaf, document), Equals("string")); AssertThat(ts_node_start_byte(leaf), Equals(string_index)); AssertThat(ts_node_end_byte(leaf), Equals(string_end_index)); AssertThat(ts_node_start_point(leaf), Equals({ 6, 4 })); AssertThat(ts_node_end_point(leaf), Equals({ 6, 7 })); - leaf = ts_node_named_descendant_for_char_range(array_node, string_index + 1, string_index + 2); + leaf = ts_node_named_descendant_for_char_range(root_node, string_index + 1, string_index + 2); AssertThat(ts_node_type(leaf, document), Equals("string")); AssertThat(ts_node_start_byte(leaf), Equals(string_index)); AssertThat(ts_node_end_byte(leaf), Equals(string_end_index)); @@ -361,7 +413,7 @@ describe("Node", []() { describe("when there is no leaf node that spans the given range", [&]() { it("returns the smallest node that does span the range", [&]() { - TSNode pair_node = ts_node_named_descendant_for_char_range(array_node, string_index, string_index + 3); + TSNode pair_node = ts_node_named_descendant_for_char_range(root_node, string_index, string_index + 3); AssertThat(ts_node_type(pair_node, document), Equals("pair")); AssertThat(ts_node_start_byte(pair_node), Equals(string_index)); AssertThat(ts_node_end_byte(pair_node), Equals(null_end_index)); @@ -370,7 +422,7 @@ describe("Node", []() { }); it("does not return invisible nodes (repeats)", [&]() { - TSNode node = ts_node_named_descendant_for_char_range(array_node, number_end_index, number_end_index + 1); + TSNode node = ts_node_named_descendant_for_char_range(root_node, number_end_index, number_end_index + 1); AssertThat(ts_node_type(node, document), Equals("array")); AssertThat(ts_node_start_byte(node), Equals(array_index)); AssertThat(ts_node_end_byte(node), Equals(array_end_index)); @@ -382,14 +434,14 @@ describe("Node", []() { describe("descendant_for_char_range(start, end)", [&]() { it("returns the smallest node that spans the given range", [&]() { - TSNode node1 = ts_node_descendant_for_char_range(array_node, colon_index, colon_index); + TSNode node1 = ts_node_descendant_for_char_range(root_node, colon_index, colon_index); AssertThat(ts_node_type(node1, document), Equals(":")); AssertThat(ts_node_start_byte(node1), Equals(colon_index)); AssertThat(ts_node_end_byte(node1), Equals(colon_index + 1)); AssertThat(ts_node_start_point(node1), Equals({ 6, 7 })); AssertThat(ts_node_end_point(node1), Equals({ 6, 8 })); - TSNode node2 = ts_node_descendant_for_char_range(array_node, string_index + 2, string_index + 4); + TSNode node2 = ts_node_descendant_for_char_range(root_node, string_index + 2, string_index + 4); AssertThat(ts_node_type(node2, document), Equals("pair")); AssertThat(ts_node_start_byte(node2), Equals(string_index)); AssertThat(ts_node_end_byte(node2), Equals(null_end_index)); @@ -402,12 +454,12 @@ describe("Node", []() { it("returns the smallest concrete node that spans the given range", [&]() { ts_document_set_input_string(document, "[\"αβγδ\", \"αβγδ\"]"); ts_document_parse(document); - TSNode array_node = ts_document_root_node(document); + TSNode root_node = ts_document_root_node(document); - TSNode node1 = ts_node_descendant_for_char_range(array_node, 7, 7); + TSNode node1 = ts_node_descendant_for_char_range(root_node, 7, 7); AssertThat(ts_node_type(node1, document), Equals(",")); - TSNode node2 = ts_node_descendant_for_byte_range(array_node, 6, 10); + TSNode node2 = ts_node_descendant_for_byte_range(root_node, 6, 10); AssertThat(ts_node_type(node2, document), Equals("string")); AssertThat(ts_node_start_byte(node2), Equals(1)); AssertThat(ts_node_end_byte(node2), Equals(11)); @@ -416,14 +468,14 @@ describe("Node", []() { describe("descendant_for_point_range(start, end)", [&]() { it("returns the smallest concrete node that spans the given range", [&]() { - TSNode node1 = ts_node_descendant_for_point_range(array_node, {6, 7}, {6, 7}); + TSNode node1 = ts_node_descendant_for_point_range(root_node, {6, 7}, {6, 7}); AssertThat(ts_node_type(node1, document), Equals(":")); AssertThat(ts_node_start_byte(node1), Equals(colon_index)); AssertThat(ts_node_end_byte(node1), Equals(colon_index + 1)); AssertThat(ts_node_start_point(node1), Equals({ 6, 7 })); AssertThat(ts_node_end_point(node1), Equals({ 6, 8 })); - TSNode node2 = ts_node_descendant_for_point_range(array_node, {6, 6}, {6, 8}); + TSNode node2 = ts_node_descendant_for_point_range(root_node, {6, 6}, {6, 8}); AssertThat(ts_node_type(node2, document), Equals("pair")); AssertThat(ts_node_start_byte(node2), Equals(string_index)); AssertThat(ts_node_end_byte(node2), Equals(null_end_index));