Simplify handling of ubiquitous tokens during reduce

spec/runtime/languages/arithmetic/errors.txt

@@ -3,7 +3,7 @@ recovers from errors at the top level
 =====================================================
 x * * y
 ---
-(expression (variable) (ERROR '*'))
+(ERROR '*')
 
 =====================================================
 recovers from errors inside parenthesized expressions

spec/runtime/languages/javascript/literals.txt

@@ -39,7 +39,7 @@ var x = {
             (statement_block (var_declaration (identifier) (identifier)))))))
 
 ==========================================
-parses comments
+parses comments. TODO - leading comments
 ==========================================
 // this is the beginning of the script.
 // here we go.
@@ -55,16 +55,14 @@ var thing = {
 };
 ---
 (program
-    (comment)
-    (comment)
-    (program (var_declaration (identifier) (object
+    (var_declaration (identifier) (object
         (comment)
         (comment)
         (identifier) (function_expression
             (formal_parameters (identifier) (comment))
             (statement_block
                 (comment)
-                (expression_statement (function_call (identifier)))))))))
+                (expression_statement (function_call (identifier))))))))
 
 ==========================================
 parses comments within expressions

spec/runtime/languages/json/errors.txt

@@ -3,7 +3,7 @@ recovers from top-level errors
 ==========================================
 [}
 ---
-(value (ERROR <EOF>) (ERROR '}'))
+(ERROR '}')
 
 ==========================================
 recovers from unexpected tokens

spec/runtime/stack_spec.cc

@@ -66,12 +66,12 @@ describe("stacks", [&]() {
 
     it("pops the given number of nodes off the stack", [&]() {
       AssertThat(stack.size, Equals<size_t>(4));
-      ts_stack_reduce(&stack, sym2, 3, hidden_symbols, 0);
+      ts_stack_reduce(&stack, sym2, 3, hidden_symbols);
       AssertThat(stack.size, Equals<size_t>(1));
     });
 
     it("returns a node with the given symbol", [&]() {
-      TSTree *node = ts_stack_reduce(&stack, sym2, 3, hidden_symbols, 0);
+      TSTree *node = ts_stack_reduce(&stack, sym2, 3, hidden_symbols);
       AssertThat(node->symbol, Equals(sym2));
     });
 
@@ -82,7 +82,7 @@ describe("stacks", [&]() {
           stack.entries[3].node,
       };
 
-      TSTree *node = ts_stack_reduce(&stack, sym2, 3, hidden_symbols, 0);
+      TSTree *node = ts_stack_reduce(&stack, sym2, 3, hidden_symbols);
       size_t child_count;
       TSTree **children = ts_tree_children(node, &child_count);
 

src/runtime/parser.c

@@ -118,7 +118,7 @@ void ts_parser_shift_extra(TSParser *parser) {
 void ts_parser_reduce(TSParser *parser, TSSymbol symbol, size_t child_count) {
   parser->next_lookahead = parser->lookahead;
   parser->lookahead = ts_stack_reduce(&parser->stack, symbol, child_count,
-                                      parser->language->hidden_symbol_flags, 1);
+                                      parser->language->hidden_symbol_flags);
 }
 
 int ts_parser_reduce_extra(TSParser *parser, TSSymbol symbol) {
@@ -177,19 +177,7 @@ int ts_parser_handle_error(TSParser *parser) {
 }
 
 TSTree *ts_parser_tree_root(TSParser *parser) {
-  TSStack *stack = &parser->stack;
-  size_t node_count = 0;
-  for (size_t i = 0; i < stack->size; i++) {
-    TSTree *node = stack->entries[i].node;
-    if (!parser->language->hidden_symbol_flags[node->symbol])
-      node_count++;
-  }
-
-  if (node_count > 1)
-    return ts_stack_reduce(stack, 2, stack->size,
-                           parser->language->hidden_symbol_flags, 0);
-  else
-    return ts_stack_top_node(stack);
+  return ts_stack_top_node(&parser->stack);
 }
 
 TSParseAction ts_parser_next_action(TSParser *parser) {

src/runtime/stack.c

@@ -52,14 +52,14 @@ size_t ts_stack_right_position(const TSStack *stack) {
 }
 
 TSTree *ts_stack_reduce(TSStack *stack, TSSymbol symbol, size_t child_count,
-                        const int *hidden_symbol_flags, int dont_count_extras) {
+                        const int *hidden_symbol_flags) {
 
   // First, walk down the stack to determine which symbols will be reduced.
   // The child node count is known ahead of time, but some children may be
-  // extra tokens, which don't count towards the child node count.
+  // ubiquitous tokens, which don't count.
   for (size_t i = 0; i < child_count; i++) {
     TSTree *child = stack->entries[stack->size - 1 - i].node;
-    if (dont_count_extras && ts_tree_is_extra(child))
+    if (ts_tree_is_extra(child))
       child_count++;
   }
 
@@ -70,7 +70,6 @@ TSTree *ts_stack_reduce(TSStack *stack, TSSymbol symbol, size_t child_count,
 
   TSTree *lookahead = ts_tree_make_node(symbol, child_count, children,
                                         hidden_symbol_flags[symbol]);
-
   ts_stack_shrink(stack, stack->size - child_count);
   return lookahead;
 }

src/runtime/stack.h

@@ -18,9 +18,7 @@ typedef struct {
 
 TSStack ts_stack_make();
 void ts_stack_delete(TSStack *);
-TSTree *ts_stack_reduce(TSStack *stack, TSSymbol symbol,
-                        size_t immediate_child_count,
-                        const int *hidden_symbol_flags, int gather_extras);
+TSTree *ts_stack_reduce(TSStack *, TSSymbol, size_t, const int *hidden_symbols);
 void ts_stack_shrink(TSStack *stack, size_t new_size);
 void ts_stack_push(TSStack *stack, TSStateId state, TSTree *node);
 TSStateId ts_stack_top_state(const TSStack *stack);