diff --git a/script/fetch-fixtures b/script/fetch-fixtures index 491ed99b..2cb9793b 100755 --- a/script/fetch-fixtures +++ b/script/fetch-fixtures @@ -23,10 +23,10 @@ fetch_grammar() { fetch_grammar 'javascript' 'origin/new-external-scanner-api' fetch_grammar 'json' 'origin/master' -fetch_grammar 'c' 'origin/master' -fetch_grammar 'cpp' 'origin/master' +fetch_grammar 'c' 'origin/use-contextual-renames' +fetch_grammar 'cpp' 'origin/use-contextual-renames' fetch_grammar 'python' 'origin/new-external-scanner-api' -fetch_grammar 'go' 'origin/master' -fetch_grammar 'ruby' 'origin/master' +fetch_grammar 'go' 'origin/use-contextual-renames' +fetch_grammar 'ruby' 'origin/new-external-scanner-api' fetch_grammar 'typescript' 'origin/master' -fetch_grammar 'bash' 'origin/new-external-scanner-api' +fetch_grammar 'bash' 'origin/master' diff --git a/src/runtime/node.c b/src/runtime/node.c index 7ddeee35..f701c047 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -33,10 +33,10 @@ static inline uint32_t ts_node__offset_row(TSNode self) { static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { const Tree *tree = ts_node__tree(self); - if (tree->context.alias_symbol > 0) { - return true; + if (include_anonymous) { + return tree->context.alias_symbol || tree->visible; } else { - return include_anonymous ? tree->visible : tree->visible && tree->named; + return tree->context.alias_is_named || (tree->visible && tree->named); } } diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 7d0d052c..a7f08a3f 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -541,17 +541,11 @@ static void parser__shift(Parser *self, StackVersion version, TSStateId state, } static bool parser__replace_children(Parser *self, Tree *tree, Tree **children, uint32_t count) { - self->scratch_tree.symbol = tree->symbol; + self->scratch_tree = *tree; self->scratch_tree.child_count = 0; ts_tree_set_children(&self->scratch_tree, count, children, self->language); if (parser__select_tree(self, tree, &self->scratch_tree)) { - tree->size = self->scratch_tree.size; - tree->padding = self->scratch_tree.padding; - tree->error_cost = self->scratch_tree.error_cost; - tree->children = self->scratch_tree.children; - tree->child_count = self->scratch_tree.child_count; - tree->named_child_count = self->scratch_tree.named_child_count; - tree->visible_child_count = self->scratch_tree.visible_child_count; + *tree = self->scratch_tree; return true; } else { return false; @@ -790,15 +784,17 @@ static bool parser__repair_error(Parser *self, StackSlice slice, array_clear(&self->reduce_actions); for (uint32_t i = 0; i < entry.action_count; i++) { - if (entry.actions[i].type == TSParseActionTypeReduce) { - TSSymbol symbol = entry.actions[i].params.symbol; - uint32_t child_count = entry.actions[i].params.child_count; + TSParseAction action = entry.actions[i]; + if (action.type == TSParseActionTypeReduce) { + TSSymbol symbol = action.params.symbol; + uint32_t child_count = action.params.child_count; if ((child_count > session.tree_count_above_error) || (child_count == session.tree_count_above_error && !ts_language_symbol_metadata(self->language, symbol).visible)) array_push(&self->reduce_actions, ((ReduceAction){ .symbol = symbol, - .count = child_count + .count = child_count, + .alias_sequence_id = action.params.alias_sequence_id, })); } } @@ -816,7 +812,6 @@ static bool parser__repair_error(Parser *self, StackSlice slice, ReduceAction repair = session.best_repair; TSStateId next_state = session.best_repair_next_state; uint32_t skip_count = session.best_repair_skip_count; - TSSymbol symbol = repair.symbol; StackSlice new_slice = array_pop(&pop.slices); TreeArray children = new_slice.trees; @@ -832,6 +827,7 @@ static bool parser__repair_error(Parser *self, StackSlice slice, TreeArray skipped_children = ts_tree_array_remove_last_n(&children, skip_count); TreeArray trailing_extras = ts_tree_array_remove_trailing_extras(&skipped_children); Tree *error = ts_tree_make_error_node(&skipped_children, self->language); + error->extra = true; array_push(&children, error); array_push_all(&children, &trailing_extras); trailing_extras.size = 0; @@ -842,8 +838,8 @@ static bool parser__repair_error(Parser *self, StackSlice slice, array_delete(&slice.trees); Tree *parent = ts_tree_make_node( - symbol, children.size, children.contents, - 0, self->language + repair.symbol, children.size, children.contents, + repair.alias_sequence_id, self->language ); parser__push(self, slice.version, parent, next_state); ts_stack_decrease_push_count(self->stack, slice.version, error->child_count); @@ -854,7 +850,7 @@ static bool parser__repair_error(Parser *self, StackSlice slice, ts_stack_halt(self->stack, slice.version); return false; } else { - LOG("repair_found sym:%s, child_count:%u, cost:%u", SYM_NAME(symbol), + LOG("repair_found sym:%s, child_count:%u, cost:%u", SYM_NAME(repair.symbol), repair.count, parent->error_cost); return true; } diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 630ee0b1..f5bcb2e4 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -50,15 +50,18 @@ Tree *ts_tree_make_leaf(TSSymbol symbol, Length padding, Length size, const TSLa .symbol = symbol, .size = size, .child_count = 0, + .children = NULL, .visible_child_count = 0, .named_child_count = 0, - .children = NULL, .alias_sequence_id = 0, .padding = padding, .visible = metadata.visible, .named = metadata.named, .has_changes = false, - .first_leaf.symbol = symbol, + .first_leaf = { + .symbol = symbol, + .lex_mode = {0, 0}, + }, .has_external_tokens = false, }; return result; @@ -170,6 +173,7 @@ void ts_tree_assign_parents(Tree *self, TreePath *path, const TSLanguage *langua child->context.alias_is_named = metadata.named; } else { child->context.alias_symbol = 0; + child->context.alias_is_named = false; } array_push(path, ((TreePathEntry){child, length_zero(), 0})); } @@ -514,9 +518,8 @@ static size_t ts_tree__write_char_to_string(char *s, size_t n, int32_t c) { return snprintf(s, n, "%d", c); } -static size_t ts_tree__write_to_string(const Tree *self, - const TSLanguage *language, char *string, - size_t limit, bool is_root, +static size_t ts_tree__write_to_string(const Tree *self, const TSLanguage *language, + char *string, size_t limit, bool is_root, bool include_all) { if (!self) return snprintf(string, limit, "(NULL)"); diff --git a/src/runtime/tree_path.h b/src/runtime/tree_path.h index efd61af5..cbddd245 100644 --- a/src/runtime/tree_path.h +++ b/src/runtime/tree_path.h @@ -137,19 +137,18 @@ TreePathComparison tree_path_compare(const TreePath *old_path, Length old_start = length_zero(); for (uint32_t i = old_path->size - 1; i + 1 > 0; i--) { old_tree = old_path->contents[i].tree; - if (old_tree->visible) { - old_start = old_path->contents[i].position; - if (i > 0) { - const TSSymbol *alias_sequence = ts_language_alias_sequence( - language, - old_path->contents[i - 1].tree->alias_sequence_id - ); - if (alias_sequence) { - old_alias_symbol = alias_sequence[old_path->contents[i].child_index]; - } + old_start = old_path->contents[i].position; + if (i > 0) { + const TSSymbol *alias_sequence = ts_language_alias_sequence( + language, + old_path->contents[i - 1].tree->alias_sequence_id + ); + if (alias_sequence) { + old_alias_symbol = alias_sequence[old_path->contents[i].child_index]; } - break; } + + if (old_tree->visible || old_alias_symbol) break; } Tree *new_tree = NULL; @@ -157,19 +156,18 @@ TreePathComparison tree_path_compare(const TreePath *old_path, Length new_start = length_zero(); for (uint32_t i = new_path->size - 1; i + 1 > 0; i--) { new_tree = new_path->contents[i].tree; - if (new_tree->visible) { - new_start = old_path->contents[i].position; - if (i > 0) { - const TSSymbol *alias_sequence = ts_language_alias_sequence( - language, - new_path->contents[i - 1].tree->alias_sequence_id - ); - if (alias_sequence) { - new_alias_symbol = alias_sequence[new_path->contents[i].child_index]; - } + new_start = old_path->contents[i].position; + if (i > 0) { + const TSSymbol *alias_sequence = ts_language_alias_sequence( + language, + new_path->contents[i - 1].tree->alias_sequence_id + ); + if (alias_sequence) { + new_alias_symbol = alias_sequence[new_path->contents[i].child_index]; } - break; } + + if (new_tree->visible || new_alias_symbol) break; } if (old_alias_symbol == new_alias_symbol) { diff --git a/test/fixtures/error_corpus/c_errors.txt b/test/fixtures/error_corpus/c_errors.txt index 459b4bfd..adaf585e 100644 --- a/test/fixtures/error_corpus/c_errors.txt +++ b/test/fixtures/error_corpus/c_errors.txt @@ -24,20 +24,20 @@ int c; (translation_unit (preproc_ifdef (identifier) - (ERROR (identifier) (identifier)) + (ERROR (type_name) (variable_name)) (comment)) - (declaration (identifier) (identifier)) + (declaration (type_name) (variable_name)) (preproc_ifdef (identifier) (ERROR (string_literal))) - (declaration (identifier) (identifier)) + (declaration (type_name) (variable_name)) (preproc_ifdef (identifier) (ERROR)) - (declaration (identifier) (identifier))) + (declaration (type_name) (variable_name))) ======================================== Errors inside blocks @@ -52,11 +52,11 @@ int main() { (translation_unit (function_definition - (identifier) - (function_declarator (identifier)) + (type_name) + (function_declarator (variable_name) (parameter_list)) (compound_statement - (declaration (identifier) (identifier)) - (ERROR (identifier) (UNEXPECTED '$'))))) + (declaration (type_name) (variable_name)) + (ERROR (variable_name) (UNEXPECTED '$'))))) ======================================== Errors inside expressions @@ -70,11 +70,11 @@ int main() { (translation_unit (function_definition - (identifier) - (function_declarator (identifier)) + (type_name) + (function_declarator (variable_name) (parameter_list)) (compound_statement - (declaration (identifier) (init_declarator - (identifier) + (declaration (type_name) (init_declarator + (variable_name) (parenthesized_expression (ERROR (number_literal)) (number_literal))))))) @@ -89,8 +89,8 @@ int y = 5; --- (translation_unit - (declaration (identifier) (ERROR (identifier)) (identifier)) - (declaration (identifier) (init_declarator (identifier) (number_literal)))) + (declaration (type_name) (ERROR (variable_name)) (variable_name)) + (declaration (type_name) (init_declarator (variable_name) (number_literal)))) ========================================== Errors at the beginnings of blocks @@ -110,25 +110,27 @@ int b() { (translation_unit (function_definition - (identifier) (function_declarator (identifier)) + (type_name) + (function_declarator (variable_name) (parameter_list)) (compound_statement - (struct_specifier (identifier)) + (struct_specifier (struct_name)) (ERROR (number_literal)) - (struct_specifier (identifier)) + (struct_specifier (struct_name)) (ERROR (number_literal)))) (function_definition - (identifier) (function_declarator (identifier)) + (type_name) + (function_declarator (variable_name) (parameter_list)) (compound_statement (declaration - (identifier) + (type_name) (init_declarator - (identifier) + (variable_name) (ERROR (identifier) (identifier)) (number_literal))) (declaration - (identifier) + (type_name) (init_declarator - (identifier) + (variable_name) (ERROR (identifier) (identifier)) (number_literal)))))) diff --git a/test/fixtures/test_grammars/renamed_inlined_rules/corpus.txt b/test/fixtures/test_grammars/aliased_inlined_rules/corpus.txt similarity index 100% rename from test/fixtures/test_grammars/renamed_inlined_rules/corpus.txt rename to test/fixtures/test_grammars/aliased_inlined_rules/corpus.txt diff --git a/test/fixtures/test_grammars/renamed_inlined_rules/grammar.json b/test/fixtures/test_grammars/aliased_inlined_rules/grammar.json similarity index 97% rename from test/fixtures/test_grammars/renamed_inlined_rules/grammar.json rename to test/fixtures/test_grammars/aliased_inlined_rules/grammar.json index 4e68cd88..02d76219 100644 --- a/test/fixtures/test_grammars/renamed_inlined_rules/grammar.json +++ b/test/fixtures/test_grammars/aliased_inlined_rules/grammar.json @@ -1,5 +1,5 @@ { - "name": "renamed_inlined_rules", + "name": "aliased_inlined_rules", "extras": [ {"type": "PATTERN", "value": "\\s"} diff --git a/test/fixtures/test_grammars/aliased_inlined_rules/readme.md b/test/fixtures/test_grammars/aliased_inlined_rules/readme.md new file mode 100644 index 00000000..9775c62b --- /dev/null +++ b/test/fixtures/test_grammars/aliased_inlined_rules/readme.md @@ -0,0 +1 @@ +This grammar shows that `ALIAS` rules can *contain* a rule that is marked as `inline`. It also shows that you can alias a rule that would otherwise be anonymous, and it will then appear as a named node. diff --git a/test/fixtures/test_grammars/inlined_renamed_rules/corpus.txt b/test/fixtures/test_grammars/aliased_rules/corpus.txt similarity index 100% rename from test/fixtures/test_grammars/inlined_renamed_rules/corpus.txt rename to test/fixtures/test_grammars/aliased_rules/corpus.txt diff --git a/test/fixtures/test_grammars/renamed_rules/grammar.json b/test/fixtures/test_grammars/aliased_rules/grammar.json similarity index 98% rename from test/fixtures/test_grammars/renamed_rules/grammar.json rename to test/fixtures/test_grammars/aliased_rules/grammar.json index 1d49df92..289fd0ea 100644 --- a/test/fixtures/test_grammars/renamed_rules/grammar.json +++ b/test/fixtures/test_grammars/aliased_rules/grammar.json @@ -1,5 +1,5 @@ { - "name": "renamed_rules", + "name": "aliased_rules", "extras": [ {"type": "PATTERN", "value": "\\s"} diff --git a/test/fixtures/test_grammars/renamed_rules/corpus.txt b/test/fixtures/test_grammars/inlined_aliased_rules/corpus.txt similarity index 100% rename from test/fixtures/test_grammars/renamed_rules/corpus.txt rename to test/fixtures/test_grammars/inlined_aliased_rules/corpus.txt diff --git a/test/fixtures/test_grammars/inlined_renamed_rules/grammar.json b/test/fixtures/test_grammars/inlined_aliased_rules/grammar.json similarity index 97% rename from test/fixtures/test_grammars/inlined_renamed_rules/grammar.json rename to test/fixtures/test_grammars/inlined_aliased_rules/grammar.json index d29f65d7..b0c121c0 100644 --- a/test/fixtures/test_grammars/inlined_renamed_rules/grammar.json +++ b/test/fixtures/test_grammars/inlined_aliased_rules/grammar.json @@ -1,5 +1,5 @@ { - "name": "inlined_renamed_rules", + "name": "inlined_aliased_rules", "extras": [ {"type": "PATTERN", "value": "\\s"} diff --git a/test/fixtures/test_grammars/inlined_renamed_rules/readme.md b/test/fixtures/test_grammars/inlined_aliased_rules/readme.md similarity index 79% rename from test/fixtures/test_grammars/inlined_renamed_rules/readme.md rename to test/fixtures/test_grammars/inlined_aliased_rules/readme.md index 952107eb..995b8630 100644 --- a/test/fixtures/test_grammars/inlined_renamed_rules/readme.md +++ b/test/fixtures/test_grammars/inlined_aliased_rules/readme.md @@ -1 +1 @@ -This grammar shows that a rule marked as `inline` can *contain* a `RENAME` rule. +This grammar shows that a rule marked as `inline` can *contain* a `ALIAS` rule. diff --git a/test/fixtures/test_grammars/renamed_inlined_rules/readme.md b/test/fixtures/test_grammars/renamed_inlined_rules/readme.md deleted file mode 100644 index d4ec6dac..00000000 --- a/test/fixtures/test_grammars/renamed_inlined_rules/readme.md +++ /dev/null @@ -1 +0,0 @@ -This grammar shows that `RENAME` rules can *contain* a rule that is marked as `inline`. It also shows that you can rename a rule that would otherwise be anonymous, and it will then appear as a named node. diff --git a/test/helpers/tree_helpers.cc b/test/helpers/tree_helpers.cc index 6c7e761d..1d94bc65 100644 --- a/test/helpers/tree_helpers.cc +++ b/test/helpers/tree_helpers.cc @@ -83,6 +83,10 @@ void assert_consistent_tree_sizes(TSNode node) { last_child_end_point = ts_node_end_point(child); } + if (actual_named_child_count != named_child_count) { + puts("UH OH"); + } + AssertThat(actual_named_child_count, Equals(named_child_count)); if (child_count > 0) { diff --git a/test/runtime/language_test.cc b/test/runtime/language_test.cc index 8ca0d64a..7d9d51cc 100644 --- a/test/runtime/language_test.cc +++ b/test/runtime/language_test.cc @@ -6,14 +6,15 @@ START_TEST describe("Language", []() { describe("symbol_name(TSSymbol symbol)", [&]() { - it("returns the correct name for renamed nodes", [&]() { + it("returns the correct name for aliased nodes", [&]() { TSCompileResult compile_result = ts_compile_grammar(R"JSON({ - "name": "renamed_rules", + "name": "aliased_rules", "rules": { "a": { - "type": "RENAME", + "type": "ALIAS", "value": "c", + "named": true, "content": { "type": "SYMBOL", "name": "b" @@ -28,7 +29,7 @@ describe("Language", []() { })JSON"); TSDocument *document = ts_document_new(); - const TSLanguage *language = load_test_language("renamed_rules", compile_result); + const TSLanguage *language = load_test_language("aliased_rules", compile_result); ts_document_set_language(document, language); ts_document_set_input_string(document, "b"); ts_document_parse(document); @@ -37,13 +38,13 @@ describe("Language", []() { char *string = ts_node_string(root_node, document); AssertThat(string, Equals("(a (c))")); - TSNode renamed_node = ts_node_child(root_node, 0); - AssertThat(ts_node_type(renamed_node, document), Equals("c")); + TSNode aliased_node = ts_node_child(root_node, 0); + AssertThat(ts_node_type(aliased_node, document), Equals("c")); - TSSymbol renamed_symbol = ts_node_symbol(renamed_node); - AssertThat(ts_language_symbol_count(language), IsGreaterThan(renamed_symbol)); - AssertThat(ts_language_symbol_name(language, renamed_symbol), Equals("c")); - AssertThat(ts_language_symbol_type(language, renamed_symbol), Equals(TSSymbolTypeRegular)); + TSSymbol aliased_symbol = ts_node_symbol(aliased_node); + AssertThat(ts_language_symbol_count(language), IsGreaterThan(aliased_symbol)); + AssertThat(ts_language_symbol_name(language, aliased_symbol), Equals("c")); + AssertThat(ts_language_symbol_type(language, aliased_symbol), Equals(TSSymbolTypeRegular)); ts_free(string); ts_document_free(document); diff --git a/test/runtime/node_test.cc b/test/runtime/node_test.cc index 3326b4d2..d458f8b3 100644 --- a/test/runtime/node_test.cc +++ b/test/runtime/node_test.cc @@ -33,8 +33,8 @@ size_t colon_index = json_string.find(":"); size_t null_index = json_string.find("null"); size_t null_end_index = null_index + string("null").size(); -string grammar_with_renames_and_extras = R"JSON({ - "name": "renames_and_extras", +string grammar_with_aliases_and_extras = R"JSON({ + "name": "aliases_and_extras", "extras": [ {"type": "PATTERN", "value": "\\s+"}, @@ -47,8 +47,9 @@ string grammar_with_renames_and_extras = R"JSON({ "members": [ {"type": "SYMBOL", "name": "b"}, { - "type": "RENAME", + "type": "ALIAS", "value": "B", + "named": true, "content": {"type": "SYMBOL", "name": "b"} }, {"type": "SYMBOL", "name": "b"} @@ -163,9 +164,9 @@ describe("Node", [&]() { AssertThat(ts_node_parent(root_node).data, Equals(nullptr)); }); - it("works correctly when the node contains renamed children and extras", [&]() { - TSCompileResult compile_result = ts_compile_grammar(grammar_with_renames_and_extras.c_str()); - const TSLanguage *language = load_test_language("renames_and_extras", compile_result); + it("works correctly when the node contains aliased children and extras", [&]() { + TSCompileResult compile_result = ts_compile_grammar(grammar_with_aliases_and_extras.c_str()); + const TSLanguage *language = load_test_language("aliases_and_extras", compile_result); ts_document_set_language(document, language); ts_document_set_input_string(document, "b ... b ... b"); ts_document_parse(document);