diff --git a/src/runtime/language.h b/src/runtime/language.h index 47850d30..386d90a9 100644 --- a/src/runtime/language.h +++ b/src/runtime/language.h @@ -63,6 +63,13 @@ ts_language_enabled_external_tokens(const TSLanguage *self, } } +static inline const TSSymbol * +ts_language_rename_sequence(const TSLanguage *self, unsigned id) { + return id > 0 ? + self->rename_sequences + id * self->max_rename_sequence_length : + NULL; +} + #ifdef __cplusplus } #endif diff --git a/src/runtime/node.c b/src/runtime/node.c index bfca082a..8179d529 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -304,7 +304,8 @@ bool ts_node_eq(TSNode self, TSNode other) { } bool ts_node_is_named(TSNode self) { - return ts_node__tree(self)->named; + const Tree *tree = ts_node__tree(self); + return tree->named || tree->context.rename_symbol != 0; } bool ts_node_has_changes(TSNode self) { diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 28f660f7..6747c26f 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -538,10 +538,11 @@ static void parser__shift(Parser *self, StackVersion version, TSStateId state, } static bool parser__switch_children(Parser *self, Tree *tree, - Tree **children, uint32_t count) { + Tree **children, uint32_t count, + const TSSymbol *rename_sequence) { self->scratch_tree.symbol = tree->symbol; self->scratch_tree.child_count = 0; - ts_tree_set_children(&self->scratch_tree, count, children); + ts_tree_set_children(&self->scratch_tree, count, children, rename_sequence); if (parser__select_tree(self, tree, &self->scratch_tree)) { tree->size = self->scratch_tree.size; tree->padding = self->scratch_tree.padding; @@ -568,6 +569,7 @@ static StackPopResult parser__reduce(Parser *self, StackVersion version, const TSLanguage *language = self->language; TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); + const TSSymbol *rename_sequence = ts_language_rename_sequence(language, rename_sequence_id); for (uint32_t i = 0; i < pop.slices.size; i++) { StackSlice slice = pop.slices.contents[i]; @@ -579,7 +581,7 @@ static StackPopResult parser__reduce(Parser *self, StackVersion version, while (child_count > 0 && slice.trees.contents[child_count - 1]->extra) child_count--; - Tree *parent = ts_tree_make_node(symbol, child_count, slice.trees.contents, metadata); + Tree *parent = ts_tree_make_node(symbol, child_count, slice.trees.contents, metadata, rename_sequence); // This pop operation may have caused multiple stack versions to collapse // into one, because they all diverged from a common state. In that case, @@ -594,7 +596,7 @@ static StackPopResult parser__reduce(Parser *self, StackVersion version, while (child_count > 0 && next_slice.trees.contents[child_count - 1]->extra) child_count--; - if (parser__switch_children(self, parent, next_slice.trees.contents, child_count)) { + if (parser__switch_children(self, parent, next_slice.trees.contents, child_count, rename_sequence)) { ts_tree_array_delete(&slice.trees); slice = next_slice; } else { @@ -839,9 +841,11 @@ static bool parser__repair_error(Parser *self, StackSlice slice, array_push(&children, slice.trees.contents[i]); array_delete(&slice.trees); - Tree *parent = - ts_tree_make_node(symbol, children.size, children.contents, - ts_language_symbol_metadata(self->language, symbol)); + Tree *parent = ts_tree_make_node( + symbol, children.size, children.contents, + ts_language_symbol_metadata(self->language, symbol), + NULL + ); parser__push(self, slice.version, parent, next_state); ts_stack_decrease_push_count(self->stack, slice.version, error->child_count); @@ -899,7 +903,11 @@ static void parser__accept(Parser *self, StackVersion version, for (uint32_t k = 0; k < child->child_count; k++) ts_tree_retain(child->children[k]); array_splice(&trees, j, 1, child->child_count, child->children); - ts_tree_set_children(root, trees.size, trees.contents); + const TSSymbol *rename_sequence = ts_language_rename_sequence( + self->language, + root->rename_sequence_id + ); + ts_tree_set_children(root, trees.size, trees.contents, rename_sequence); ts_tree_release(child); break; } diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 793deaa3..f3163c26 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -7,6 +7,7 @@ #include "runtime/alloc.h" #include "runtime/tree.h" #include "runtime/length.h" +#include "runtime/language.h" #include "runtime/error_costs.h" TSStateId TS_TREE_STATE_NONE = USHRT_MAX; @@ -127,16 +128,15 @@ void ts_tree_assign_parents(Tree *self, TreePath *path, const TSLanguage *langua while (path->size > 0) { Tree *tree = array_pop(path).tree; Length offset = length_zero(); - const TSSymbol *rename_symbols = language->rename_sequences + - tree->rename_sequence_id * language->max_rename_sequence_length; + const TSSymbol *rename_sequence = ts_language_rename_sequence(language, tree->rename_sequence_id); for (uint32_t i = 0; i < tree->child_count; i++) { Tree *child = tree->children[i]; if (child->context.parent != tree || child->context.index != i) { child->context.parent = tree; child->context.index = i; child->context.offset = offset; - if (tree->rename_sequence_id && rename_symbols[i] != 0) { - child->context.rename_symbol = rename_symbols[i]; + if (rename_sequence && rename_sequence[i] != 0) { + child->context.rename_symbol = rename_sequence[i]; } array_push(path, ((TreePathEntry){child, length_zero(), 0})); } @@ -146,7 +146,8 @@ void ts_tree_assign_parents(Tree *self, TreePath *path, const TSLanguage *langua } -void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children) { +void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children, + const TSSymbol *rename_sequence) { if (self->child_count > 0) ts_free(self->children); @@ -176,8 +177,9 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children) { if (child->visible) { self->visible_child_count++; - if (child->named) + if (child->named || (rename_sequence && rename_sequence[i] != 0)) { self->named_child_count++; + } } else if (child->child_count > 0) { self->visible_child_count += child->visible_child_count; self->named_child_count += child->named_child_count; @@ -208,11 +210,11 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children) { } } -Tree *ts_tree_make_node(TSSymbol symbol, uint32_t child_count, - Tree **children, TSSymbolMetadata metadata) { +Tree *ts_tree_make_node(TSSymbol symbol, uint32_t child_count, Tree **children, + TSSymbolMetadata metadata, const TSSymbol *rename_sequence) { Tree *result = ts_tree_make_leaf(symbol, length_zero(), length_zero(), metadata); - ts_tree_set_children(result, child_count, children); + ts_tree_set_children(result, child_count, children, rename_sequence); return result; } @@ -230,7 +232,7 @@ Tree *ts_tree_make_error_node(TreeArray *children) { Tree *result = ts_tree_make_node( ts_builtin_sym_error, children->size, children->contents, - (TSSymbolMetadata){.extra = false, .visible = true, .named = true }); + (TSSymbolMetadata){.extra = false, .visible = true, .named = true }, NULL); result->fragile_left = true; result->fragile_right = true; @@ -482,7 +484,11 @@ static size_t ts_tree__write_to_string(const Tree *self, char *cursor = string; char **writer = (limit > 0) ? &cursor : &string; - bool visible = include_all || is_root || (self->visible && self->named); + bool visible = + include_all || + is_root || + (self->visible && self->named) || + self->context.rename_symbol != 0; if (visible && !is_root) { cursor += snprintf(*writer, limit, " "); diff --git a/src/runtime/tree.h b/src/runtime/tree.h index ab5f050b..3b58d86b 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -75,7 +75,7 @@ TreeArray ts_tree_array_remove_last_n(TreeArray *, uint32_t); TreeArray ts_tree_array_remove_trailing_extras(TreeArray *); Tree *ts_tree_make_leaf(TSSymbol, Length, Length, TSSymbolMetadata); -Tree *ts_tree_make_node(TSSymbol, uint32_t, Tree **, TSSymbolMetadata); +Tree *ts_tree_make_node(TSSymbol, uint32_t, Tree **, TSSymbolMetadata, const TSSymbol *); Tree *ts_tree_make_copy(Tree *child); Tree *ts_tree_make_error_node(TreeArray *); Tree *ts_tree_make_error(Length, Length, int32_t); @@ -86,7 +86,7 @@ int ts_tree_compare(const Tree *tree1, const Tree *tree2); uint32_t ts_tree_start_column(const Tree *self); uint32_t ts_tree_end_column(const Tree *self); -void ts_tree_set_children(Tree *, uint32_t, Tree **); +void ts_tree_set_children(Tree *, uint32_t, Tree **, const TSSymbol *); void ts_tree_assign_parents(Tree *, TreePath *, const TSLanguage *); void ts_tree_edit(Tree *, const TSInputEdit *edit); char *ts_tree_string(const Tree *, const TSLanguage *, bool include_all); diff --git a/test/fixtures/test_grammars/inlined_renamed_rules/corpus.txt b/test/fixtures/test_grammars/inlined_renamed_rules/corpus.txt new file mode 100644 index 00000000..8789179d --- /dev/null +++ b/test/fixtures/test_grammars/inlined_renamed_rules/corpus.txt @@ -0,0 +1,18 @@ +====================================== +Method calls +====================================== + +a.b(c(d.e)); + +--- + +(statement + (call_expression + (member_expression + (variable_name) + (property_name)) + (call_expression + (variable_name) + (member_expression + (variable_name) + (property_name))))) diff --git a/test/fixtures/test_grammars/inlined_renamed_rules/grammar.json b/test/fixtures/test_grammars/inlined_renamed_rules/grammar.json new file mode 100644 index 00000000..1e399941 --- /dev/null +++ b/test/fixtures/test_grammars/inlined_renamed_rules/grammar.json @@ -0,0 +1,73 @@ +{ + "name": "inlined_renamed_rules", + + "extras": [ + {"type": "PATTERN", "value": "\\s"} + ], + + "inline": [ + "expression" + ], + + "rules": { + "statement": { + "type": "SEQ", + "members": [ + {"type": "SYMBOL", "name": "expression"}, + {"type": "STRING", "value": ";"} + ] + }, + + "expression": { + "type": "CHOICE", + "members": [ + {"type": "SYMBOL", "name": "call_expression"}, + {"type": "SYMBOL", "name": "member_expression"}, + { + "type": "RENAME", + "value": "variable_name", + "content": { + "type": "SYMBOL", + "name": "identifier" + } + } + ] + }, + + "call_expression": { + "type": "PREC_LEFT", + "value": 0, + "content": { + "type": "SEQ", + "members": [ + {"type": "SYMBOL", "name": "expression"}, + {"type": "STRING", "value": "("}, + {"type": "SYMBOL", "name": "expression"}, + {"type": "STRING", "value": ")"}, + ] + } + }, + + "member_expression": { + "type": "PREC_LEFT", + "value": 1, + "content": { + "type": "SEQ", + "members": [ + {"type": "SYMBOL", "name": "expression"}, + {"type": "STRING", "value": "."}, + { + "type": "RENAME", + "value": "property_name", + "content": { + "type": "SYMBOL", + "name": "identifier" + } + } + ] + } + }, + + "identifier": {"type": "PATTERN", "value": "\\a+"} + } +} diff --git a/test/fixtures/test_grammars/inlined_renamed_rules/readme.md b/test/fixtures/test_grammars/inlined_renamed_rules/readme.md new file mode 100644 index 00000000..952107eb --- /dev/null +++ b/test/fixtures/test_grammars/inlined_renamed_rules/readme.md @@ -0,0 +1 @@ +This grammar shows that a rule marked as `inline` can *contain* a `RENAME` rule. diff --git a/test/fixtures/test_grammars/renamed_inlined_rules/corpus.txt b/test/fixtures/test_grammars/renamed_inlined_rules/corpus.txt new file mode 100644 index 00000000..49a91158 --- /dev/null +++ b/test/fixtures/test_grammars/renamed_inlined_rules/corpus.txt @@ -0,0 +1,13 @@ +========================= +OK +========================= + +a.b.c + +--- + +(expression (member_expression + (expression (member_expression + (expression (variable_name)) + (property_name))) + (property_name))) diff --git a/test/fixtures/test_grammars/renamed_inlined_rules/grammar.json b/test/fixtures/test_grammars/renamed_inlined_rules/grammar.json new file mode 100644 index 00000000..6dfb4ebf --- /dev/null +++ b/test/fixtures/test_grammars/renamed_inlined_rules/grammar.json @@ -0,0 +1,57 @@ +{ + "name": "renamed_inlined_rules", + + "extras": [ + {"type": "PATTERN", "value": "\\s"} + ], + + "inline": [ + "identifier" + ], + + "rules": { + "expression": { + "type": "CHOICE", + "members": [ + {"type": "SYMBOL", "name": "member_expression"}, + { + "type": "RENAME", + "value": "variable_name", + "content": { + "type": "SYMBOL", + "name": "identifier" + } + } + ] + }, + + "member_expression": { + "type": "PREC_LEFT", + "value": 1, + "content": { + "type": "SEQ", + "members": [ + {"type": "SYMBOL", "name": "expression"}, + {"type": "STRING", "value": "."}, + { + "type": "RENAME", + "value": "property_name", + "content": { + "type": "SYMBOL", + "name": "identifier" + } + } + ] + } + }, + + "identifier": { + "type": "CHOICE", + "members": [ + {"type": "STRING", "value": "a"}, + {"type": "STRING", "value": "b"}, + {"type": "STRING", "value": "c"} + ] + } + } +} diff --git a/test/fixtures/test_grammars/renamed_inlined_rules/readme.md b/test/fixtures/test_grammars/renamed_inlined_rules/readme.md new file mode 100644 index 00000000..d4ec6dac --- /dev/null +++ b/test/fixtures/test_grammars/renamed_inlined_rules/readme.md @@ -0,0 +1 @@ +This grammar shows that `RENAME` rules can *contain* a rule that is marked as `inline`. It also shows that you can rename a rule that would otherwise be anonymous, and it will then appear as a named node. diff --git a/test/helpers/tree_helpers.cc b/test/helpers/tree_helpers.cc index 8b6b6623..6c7e761d 100644 --- a/test/helpers/tree_helpers.cc +++ b/test/helpers/tree_helpers.cc @@ -53,11 +53,11 @@ bool operator==(const std::vector &vec, const TreeArray &array) { void assert_consistent_tree_sizes(TSNode node) { size_t child_count = ts_node_child_count(node); + size_t named_child_count = ts_node_named_child_count(node); size_t start_byte = ts_node_start_byte(node); size_t end_byte = ts_node_end_byte(node); TSPoint start_point = ts_node_start_point(node); TSPoint end_point = ts_node_end_point(node); - bool some_child_has_changes = false; AssertThat(start_byte, !IsGreaterThan(end_byte)); AssertThat(start_point, !IsGreaterThan(end_point)); @@ -65,6 +65,8 @@ void assert_consistent_tree_sizes(TSNode node) { size_t last_child_end_byte = start_byte; TSPoint last_child_end_point = start_point; + bool some_child_has_changes = false; + size_t actual_named_child_count = 0; for (size_t i = 0; i < child_count; i++) { TSNode child = ts_node_child(node, i); size_t child_start_byte = ts_node_start_byte(child); @@ -73,13 +75,16 @@ void assert_consistent_tree_sizes(TSNode node) { AssertThat(child_start_byte, !IsLessThan(last_child_end_byte)); AssertThat(child_start_point, !IsLessThan(last_child_end_point)); assert_consistent_tree_sizes(child); - if (ts_node_has_changes(child)) - some_child_has_changes = true; + + if (ts_node_has_changes(child)) some_child_has_changes = true; + if (ts_node_is_named(child)) actual_named_child_count++; last_child_end_byte = ts_node_end_byte(child); last_child_end_point = ts_node_end_point(child); } + AssertThat(actual_named_child_count, Equals(named_child_count)); + if (child_count > 0) { AssertThat(end_byte, !IsLessThan(last_child_end_byte)); AssertThat(end_point, !IsLessThan(last_child_end_point)); diff --git a/test/integration/test_grammars.cc b/test/integration/test_grammars.cc index c08af27c..e0d78eae 100644 --- a/test/integration/test_grammars.cc +++ b/test/integration/test_grammars.cc @@ -3,6 +3,7 @@ #include "helpers/load_language.h" #include "helpers/stderr_logger.h" #include "helpers/file_helpers.h" +#include "helpers/tree_helpers.h" #include "runtime/alloc.h" START_TEST @@ -54,6 +55,7 @@ for (auto &language_name : test_languages) { ts_document_parse(document); TSNode root_node = ts_document_root_node(document); + assert_consistent_tree_sizes(root_node); const char *node_string = ts_node_string(root_node, document); string result(node_string); ts_free((void *)node_string); diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc index d751c717..40286ac7 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/tree_test.cc @@ -72,7 +72,7 @@ describe("Tree", []() { parent1 = ts_tree_make_node(symbol3, 2, tree_array({ tree1, tree2, - }), visible); + }), visible, nullptr); }); after_each([&]() { @@ -103,7 +103,7 @@ describe("Tree", []() { parent = ts_tree_make_node(symbol3, 2, tree_array({ tree1, tree2, - }), visible); + }), visible, nullptr); }); after_each([&]() { @@ -127,7 +127,7 @@ describe("Tree", []() { parent = ts_tree_make_node(symbol3, 2, tree_array({ tree1, tree2, - }), visible); + }), visible, nullptr); }); after_each([&]() { @@ -151,7 +151,7 @@ describe("Tree", []() { parent = ts_tree_make_node(symbol3, 2, tree_array({ tree1, tree2, - }), visible); + }), visible, nullptr); }); after_each([&]() { @@ -173,7 +173,7 @@ describe("Tree", []() { ts_tree_make_leaf(symbol2, {2, 2, {0, 2}}, {3, 3, {0, 3}}, visible), ts_tree_make_leaf(symbol3, {2, 2, {0, 2}}, {3, 3, {0, 3}}, visible), ts_tree_make_leaf(symbol4, {2, 2, {0, 2}}, {3, 3, {0, 3}}, visible), - }), visible); + }), visible, nullptr); AssertThat(tree->padding, Equals({2, 2, {0, 2}})); AssertThat(tree->size, Equals({13, 13, {0, 13}})); @@ -350,14 +350,14 @@ describe("Tree", []() { Tree *parent = ts_tree_make_node(symbol2, 2, tree_array({ leaf, leaf_copy, - }), visible); + }), visible, nullptr); ts_tree_retain(leaf); ts_tree_retain(leaf_copy); Tree *parent_copy = ts_tree_make_node(symbol2, 2, tree_array({ leaf, leaf_copy, - }), visible); + }), visible, nullptr); ts_tree_retain(leaf); ts_tree_retain(leaf_copy); @@ -401,14 +401,14 @@ describe("Tree", []() { Tree *parent = ts_tree_make_node(symbol2, 2, tree_array({ leaf, leaf2, - }), visible); + }), visible, nullptr); ts_tree_retain(leaf); ts_tree_retain(leaf2); Tree *different_parent = ts_tree_make_node(symbol2, 2, tree_array({ leaf2, leaf, - }), visible); + }), visible, nullptr); ts_tree_retain(leaf2); ts_tree_retain(leaf); @@ -438,14 +438,14 @@ describe("Tree", []() { (tree3 = make_external(ts_tree_make_leaf(symbol3, padding, size, visible))), (tree4 = ts_tree_make_leaf(symbol4, padding, size, visible)), (tree5 = ts_tree_make_leaf(symbol5, padding, size, visible)), - }), visible)), + }), visible, nullptr)), (tree6 = ts_tree_make_node(symbol6, 2, tree_array({ (tree7 = ts_tree_make_node(symbol7, 1, tree_array({ (tree8 = ts_tree_make_leaf(symbol8, padding, size, visible)), - }), visible)), + }), visible, nullptr)), (tree9 = ts_tree_make_leaf(symbol9, padding, size, visible)), - }), visible)), - }), visible); + }), visible, nullptr)), + }), visible, nullptr); auto token = ts_tree_last_external_token(tree1); AssertThat(token, Equals(tree3));