Get tests passing w/ new alias API

script/fetch-fixtures

@@ -23,10 +23,10 @@ fetch_grammar() {
 
 fetch_grammar 'javascript' 'origin/new-external-scanner-api'
 fetch_grammar 'json'       'origin/master'
-fetch_grammar 'c'          'origin/master'
-fetch_grammar 'cpp'        'origin/master'
+fetch_grammar 'c'          'origin/use-contextual-renames'
+fetch_grammar 'cpp'        'origin/use-contextual-renames'
 fetch_grammar 'python'     'origin/new-external-scanner-api'
-fetch_grammar 'go'         'origin/master'
-fetch_grammar 'ruby'       'origin/master'
+fetch_grammar 'go'         'origin/use-contextual-renames'
+fetch_grammar 'ruby'       'origin/new-external-scanner-api'
 fetch_grammar 'typescript' 'origin/master'
-fetch_grammar 'bash'       'origin/new-external-scanner-api'
+fetch_grammar 'bash'       'origin/master'

src/runtime/node.c

@@ -33,10 +33,10 @@ static inline uint32_t ts_node__offset_row(TSNode self) {
 
 static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) {
   const Tree *tree = ts_node__tree(self);
-  if (tree->context.alias_symbol > 0) {
-    return true;
+  if (include_anonymous) {
+    return tree->context.alias_symbol || tree->visible;
   } else {
-    return include_anonymous ? tree->visible : tree->visible && tree->named;
+    return tree->context.alias_is_named || (tree->visible && tree->named);
   }
 }
 

src/runtime/parser.c

@@ -541,17 +541,11 @@ static void parser__shift(Parser *self, StackVersion version, TSStateId state,
 }
 
 static bool parser__replace_children(Parser *self, Tree *tree, Tree **children, uint32_t count) {
-  self->scratch_tree.symbol = tree->symbol;
+  self->scratch_tree = *tree;
   self->scratch_tree.child_count = 0;
   ts_tree_set_children(&self->scratch_tree, count, children, self->language);
   if (parser__select_tree(self, tree, &self->scratch_tree)) {
-    tree->size = self->scratch_tree.size;
-    tree->padding = self->scratch_tree.padding;
-    tree->error_cost = self->scratch_tree.error_cost;
-    tree->children = self->scratch_tree.children;
-    tree->child_count = self->scratch_tree.child_count;
-    tree->named_child_count = self->scratch_tree.named_child_count;
-    tree->visible_child_count = self->scratch_tree.visible_child_count;
+    *tree = self->scratch_tree;
     return true;
   } else {
     return false;
@@ -790,15 +784,17 @@ static bool parser__repair_error(Parser *self, StackSlice slice,
 
   array_clear(&self->reduce_actions);
   for (uint32_t i = 0; i < entry.action_count; i++) {
-    if (entry.actions[i].type == TSParseActionTypeReduce) {
-      TSSymbol symbol = entry.actions[i].params.symbol;
-      uint32_t child_count = entry.actions[i].params.child_count;
+    TSParseAction action = entry.actions[i];
+    if (action.type == TSParseActionTypeReduce) {
+      TSSymbol symbol = action.params.symbol;
+      uint32_t child_count = action.params.child_count;
       if ((child_count > session.tree_count_above_error) ||
           (child_count == session.tree_count_above_error &&
            !ts_language_symbol_metadata(self->language, symbol).visible))
         array_push(&self->reduce_actions, ((ReduceAction){
           .symbol = symbol,
-          .count = child_count
+          .count = child_count,
+          .alias_sequence_id = action.params.alias_sequence_id,
         }));
     }
   }
@@ -816,7 +812,6 @@ static bool parser__repair_error(Parser *self, StackSlice slice,
   ReduceAction repair = session.best_repair;
   TSStateId next_state = session.best_repair_next_state;
   uint32_t skip_count = session.best_repair_skip_count;
-  TSSymbol symbol = repair.symbol;
 
   StackSlice new_slice = array_pop(&pop.slices);
   TreeArray children = new_slice.trees;
@@ -832,6 +827,7 @@ static bool parser__repair_error(Parser *self, StackSlice slice,
   TreeArray skipped_children = ts_tree_array_remove_last_n(&children, skip_count);
   TreeArray trailing_extras = ts_tree_array_remove_trailing_extras(&skipped_children);
   Tree *error = ts_tree_make_error_node(&skipped_children, self->language);
+  error->extra = true;
   array_push(&children, error);
   array_push_all(&children, &trailing_extras);
   trailing_extras.size = 0;
@@ -842,8 +838,8 @@ static bool parser__repair_error(Parser *self, StackSlice slice,
   array_delete(&slice.trees);
 
   Tree *parent = ts_tree_make_node(
-    symbol, children.size, children.contents,
-    0, self->language
+    repair.symbol, children.size, children.contents,
+    repair.alias_sequence_id, self->language
   );
   parser__push(self, slice.version, parent, next_state);
   ts_stack_decrease_push_count(self->stack, slice.version, error->child_count);
@@ -854,7 +850,7 @@ static bool parser__repair_error(Parser *self, StackSlice slice,
     ts_stack_halt(self->stack, slice.version);
     return false;
   } else {
-    LOG("repair_found sym:%s, child_count:%u, cost:%u", SYM_NAME(symbol),
+    LOG("repair_found sym:%s, child_count:%u, cost:%u", SYM_NAME(repair.symbol),
         repair.count, parent->error_cost);
     return true;
   }

src/runtime/tree.c

@@ -50,15 +50,18 @@ Tree *ts_tree_make_leaf(TSSymbol symbol, Length padding, Length size, const TSLa
     .symbol = symbol,
     .size = size,
     .child_count = 0,
+    .children = NULL,
     .visible_child_count = 0,
     .named_child_count = 0,
-    .children = NULL,
     .alias_sequence_id = 0,
     .padding = padding,
     .visible = metadata.visible,
     .named = metadata.named,
     .has_changes = false,
-    .first_leaf.symbol = symbol,
+    .first_leaf = {
+      .symbol = symbol,
+      .lex_mode = {0, 0},
+    },
     .has_external_tokens = false,
   };
   return result;
@@ -170,6 +173,7 @@ void ts_tree_assign_parents(Tree *self, TreePath *path, const TSLanguage *langua
           child->context.alias_is_named = metadata.named;
         } else {
           child->context.alias_symbol = 0;
+          child->context.alias_is_named = false;
         }
         array_push(path, ((TreePathEntry){child, length_zero(), 0}));
       }
@@ -514,9 +518,8 @@ static size_t ts_tree__write_char_to_string(char *s, size_t n, int32_t c) {
     return snprintf(s, n, "%d", c);
 }
 
-static size_t ts_tree__write_to_string(const Tree *self,
-                                       const TSLanguage *language, char *string,
-                                       size_t limit, bool is_root,
+static size_t ts_tree__write_to_string(const Tree *self, const TSLanguage *language,
+                                       char *string, size_t limit, bool is_root,
                                        bool include_all) {
   if (!self) return snprintf(string, limit, "(NULL)");
 

src/runtime/tree_path.h

@@ -137,19 +137,18 @@ TreePathComparison tree_path_compare(const TreePath *old_path,
   Length old_start = length_zero();
   for (uint32_t i = old_path->size - 1; i + 1 > 0; i--) {
     old_tree = old_path->contents[i].tree;
-    if (old_tree->visible) {
-      old_start = old_path->contents[i].position;
-      if (i > 0) {
-        const TSSymbol *alias_sequence = ts_language_alias_sequence(
-          language,
-          old_path->contents[i - 1].tree->alias_sequence_id
-        );
-        if (alias_sequence) {
-          old_alias_symbol = alias_sequence[old_path->contents[i].child_index];
-        }
+    old_start = old_path->contents[i].position;
+    if (i > 0) {
+      const TSSymbol *alias_sequence = ts_language_alias_sequence(
+        language,
+        old_path->contents[i - 1].tree->alias_sequence_id
+      );
+      if (alias_sequence) {
+        old_alias_symbol = alias_sequence[old_path->contents[i].child_index];
       }
-      break;
     }
+
+    if (old_tree->visible || old_alias_symbol) break;
   }
 
   Tree *new_tree = NULL;
@@ -157,19 +156,18 @@ TreePathComparison tree_path_compare(const TreePath *old_path,
   Length new_start = length_zero();
   for (uint32_t i = new_path->size - 1; i + 1 > 0; i--) {
     new_tree = new_path->contents[i].tree;
-    if (new_tree->visible) {
-      new_start = old_path->contents[i].position;
-      if (i > 0) {
-        const TSSymbol *alias_sequence = ts_language_alias_sequence(
-          language,
-          new_path->contents[i - 1].tree->alias_sequence_id
-        );
-        if (alias_sequence) {
-          new_alias_symbol = alias_sequence[new_path->contents[i].child_index];
-        }
+    new_start = old_path->contents[i].position;
+    if (i > 0) {
+      const TSSymbol *alias_sequence = ts_language_alias_sequence(
+        language,
+        new_path->contents[i - 1].tree->alias_sequence_id
+      );
+      if (alias_sequence) {
+        new_alias_symbol = alias_sequence[new_path->contents[i].child_index];
       }
-      break;
     }
+
+    if (new_tree->visible || new_alias_symbol) break;
   }
 
   if (old_alias_symbol == new_alias_symbol) {

test/fixtures/error_corpus/c_errors.txt

@@ -24,20 +24,20 @@ int c;
 
 (translation_unit
   (preproc_ifdef (identifier)
-    (ERROR (identifier) (identifier))
+    (ERROR (type_name) (variable_name))
     (comment))
 
-  (declaration (identifier) (identifier))
+  (declaration (type_name) (variable_name))
 
   (preproc_ifdef (identifier)
     (ERROR (string_literal)))
 
-  (declaration (identifier) (identifier))
+  (declaration (type_name) (variable_name))
 
   (preproc_ifdef (identifier)
     (ERROR))
 
-  (declaration (identifier) (identifier)))
+  (declaration (type_name) (variable_name)))
 
 ========================================
 Errors inside blocks
@@ -52,11 +52,11 @@ int main() {
 
 (translation_unit
   (function_definition
-    (identifier)
-    (function_declarator (identifier))
+    (type_name)
+    (function_declarator (variable_name) (parameter_list))
     (compound_statement
-      (declaration (identifier) (identifier))
-      (ERROR (identifier) (UNEXPECTED '$')))))
+      (declaration (type_name) (variable_name))
+      (ERROR (variable_name) (UNEXPECTED '$')))))
 
 ========================================
 Errors inside expressions
@@ -70,11 +70,11 @@ int main() {
 
 (translation_unit
   (function_definition
-    (identifier)
-    (function_declarator (identifier))
+    (type_name)
+    (function_declarator (variable_name) (parameter_list))
     (compound_statement
-      (declaration (identifier) (init_declarator
-        (identifier)
+      (declaration (type_name) (init_declarator
+        (variable_name)
         (parenthesized_expression
           (ERROR (number_literal))
           (number_literal)))))))
@@ -89,8 +89,8 @@ int y = 5;
 ---
 
 (translation_unit
-  (declaration (identifier) (ERROR (identifier)) (identifier))
-  (declaration (identifier) (init_declarator (identifier) (number_literal))))
+  (declaration (type_name) (ERROR (variable_name)) (variable_name))
+  (declaration (type_name) (init_declarator (variable_name) (number_literal))))
 
 ==========================================
 Errors at the beginnings of blocks
@@ -110,25 +110,27 @@ int b() {
 
 (translation_unit
   (function_definition
-    (identifier) (function_declarator (identifier))
+    (type_name)
+    (function_declarator (variable_name) (parameter_list))
     (compound_statement
-      (struct_specifier (identifier))
+      (struct_specifier (struct_name))
       (ERROR (number_literal))
-      (struct_specifier (identifier))
+      (struct_specifier (struct_name))
       (ERROR (number_literal))))
 
   (function_definition
-    (identifier) (function_declarator (identifier))
+    (type_name)
+    (function_declarator (variable_name) (parameter_list))
     (compound_statement
       (declaration
-        (identifier)
+        (type_name)
         (init_declarator
-          (identifier)
+          (variable_name)
           (ERROR (identifier) (identifier))
           (number_literal)))
       (declaration
-        (identifier)
+        (type_name)
         (init_declarator
-          (identifier)
+          (variable_name)
           (ERROR (identifier) (identifier))
           (number_literal))))))

test/fixtures/test_grammars/aliased_inlined_rules/grammar.json

@@ -1,5 +1,5 @@
 {
-  "name": "renamed_inlined_rules",
+  "name": "aliased_inlined_rules",
 
   "extras": [
     {"type": "PATTERN", "value": "\\s"}

test/fixtures/test_grammars/aliased_inlined_rules/readme.md

@@ -0,0 +1 @@
+This grammar shows that `ALIAS` rules can *contain* a rule that is marked as `inline`. It also shows that you can alias a rule that would otherwise be anonymous, and it will then appear as a named node.

test/fixtures/test_grammars/aliased_rules/grammar.json

@@ -1,5 +1,5 @@
 {
-  "name": "renamed_rules",
+  "name": "aliased_rules",
 
   "extras": [
     {"type": "PATTERN", "value": "\\s"}

test/fixtures/test_grammars/inlined_aliased_rules/grammar.json

@@ -1,5 +1,5 @@
 {
-  "name": "inlined_renamed_rules",
+  "name": "inlined_aliased_rules",
 
   "extras": [
     {"type": "PATTERN", "value": "\\s"}

test/fixtures/test_grammars/inlined_aliased_rules/readme.md

@@ -1 +1 @@
-This grammar shows that a rule marked as `inline` can *contain* a `RENAME` rule.
+This grammar shows that a rule marked as `inline` can *contain* a `ALIAS` rule.

test/fixtures/test_grammars/renamed_inlined_rules/readme.md

@@ -1 +0,0 @@
-This grammar shows that `RENAME` rules can *contain* a rule that is marked as `inline`. It also shows that you can rename a rule that would otherwise be anonymous, and it will then appear as a named node.

test/helpers/tree_helpers.cc

@@ -83,6 +83,10 @@ void assert_consistent_tree_sizes(TSNode node) {
     last_child_end_point = ts_node_end_point(child);
   }
 
+  if (actual_named_child_count != named_child_count) {
+    puts("UH OH");
+  }
+
   AssertThat(actual_named_child_count, Equals(named_child_count));
 
   if (child_count > 0) {

test/runtime/language_test.cc

@@ -6,14 +6,15 @@ START_TEST
 
 describe("Language", []() {
   describe("symbol_name(TSSymbol symbol)", [&]() {
-    it("returns the correct name for renamed nodes", [&]() {
+    it("returns the correct name for aliased nodes", [&]() {
       TSCompileResult compile_result = ts_compile_grammar(R"JSON({
-        "name": "renamed_rules",
+        "name": "aliased_rules",
 
         "rules": {
           "a": {
-            "type": "RENAME",
+            "type": "ALIAS",
             "value": "c",
+            "named": true,
             "content": {
               "type": "SYMBOL",
               "name": "b"
@@ -28,7 +29,7 @@ describe("Language", []() {
       })JSON");
 
       TSDocument *document = ts_document_new();
-      const TSLanguage *language =  load_test_language("renamed_rules", compile_result);
+      const TSLanguage *language =  load_test_language("aliased_rules", compile_result);
       ts_document_set_language(document, language);
       ts_document_set_input_string(document, "b");
       ts_document_parse(document);
@@ -37,13 +38,13 @@ describe("Language", []() {
       char *string = ts_node_string(root_node, document);
       AssertThat(string, Equals("(a (c))"));
 
-      TSNode renamed_node = ts_node_child(root_node, 0);
-      AssertThat(ts_node_type(renamed_node, document), Equals("c"));
+      TSNode aliased_node = ts_node_child(root_node, 0);
+      AssertThat(ts_node_type(aliased_node, document), Equals("c"));
 
-      TSSymbol renamed_symbol = ts_node_symbol(renamed_node);
-      AssertThat(ts_language_symbol_count(language), IsGreaterThan(renamed_symbol));
-      AssertThat(ts_language_symbol_name(language, renamed_symbol), Equals("c"));
-      AssertThat(ts_language_symbol_type(language, renamed_symbol), Equals(TSSymbolTypeRegular));
+      TSSymbol aliased_symbol = ts_node_symbol(aliased_node);
+      AssertThat(ts_language_symbol_count(language), IsGreaterThan(aliased_symbol));
+      AssertThat(ts_language_symbol_name(language, aliased_symbol), Equals("c"));
+      AssertThat(ts_language_symbol_type(language, aliased_symbol), Equals(TSSymbolTypeRegular));
 
       ts_free(string);
       ts_document_free(document);

test/runtime/node_test.cc

@@ -33,8 +33,8 @@ size_t colon_index = json_string.find(":");
 size_t null_index = json_string.find("null");
 size_t null_end_index = null_index + string("null").size();
 
-string grammar_with_renames_and_extras = R"JSON({
-  "name": "renames_and_extras",
+string grammar_with_aliases_and_extras = R"JSON({
+  "name": "aliases_and_extras",
 
   "extras": [
     {"type": "PATTERN", "value": "\\s+"},
@@ -47,8 +47,9 @@ string grammar_with_renames_and_extras = R"JSON({
       "members": [
         {"type": "SYMBOL", "name": "b"},
         {
-          "type": "RENAME",
+          "type": "ALIAS",
           "value": "B",
+          "named": true,
           "content": {"type": "SYMBOL", "name": "b"}
         },
         {"type": "SYMBOL", "name": "b"}
@@ -163,9 +164,9 @@ describe("Node", [&]() {
       AssertThat(ts_node_parent(root_node).data, Equals<void *>(nullptr));
     });
 
-    it("works correctly when the node contains renamed children and extras", [&]() {
-      TSCompileResult compile_result = ts_compile_grammar(grammar_with_renames_and_extras.c_str());
-      const TSLanguage *language =  load_test_language("renames_and_extras", compile_result);
+    it("works correctly when the node contains aliased children and extras", [&]() {
+      TSCompileResult compile_result = ts_compile_grammar(grammar_with_aliases_and_extras.c_str());
+      const TSLanguage *language =  load_test_language("aliases_and_extras", compile_result);
       ts_document_set_language(document, language);
       ts_document_set_input_string(document, "b ... b ... b");
       ts_document_parse(document);