Incrementally build a tree of skipped tokens

Rather than pushing them to the stack individually

src/compiler/build_tables/parse_table_builder.cc

@@ -150,6 +150,8 @@ class ParseTableBuilderImpl : public ParseTableBuilder {
       MatchesLongerStringWithValidNextChar
     );
 
+    parse_table.states[state_id].terminal_entries.clear();
+
     // Add all the tokens that have no conflict with other tokens.
     LookaheadSet non_conflicting_tokens;
     for (unsigned i = 0; i < lexical_grammar.variables.size(); i++) {
@@ -186,12 +188,6 @@ class ParseTableBuilderImpl : public ParseTableBuilder {
       }
     }
 
-    for (const Symbol &symbol : grammar.extra_tokens) {
-      if (!parse_table.states[state_id].terminal_entries.count(symbol)) {
-        parse_table.add_terminal_action(state_id, symbol, ParseAction::ShiftExtra());
-      }
-    }
-
     for (size_t i = 0; i < grammar.external_tokens.size(); i++) {
       if (grammar.external_tokens[i].corresponding_internal_token == rules::NONE()) {
         parse_table.states[state_id].terminal_entries[Symbol::external(i)].actions.push_back(ParseAction::Recover());

src/runtime/language.c

@@ -4,7 +4,7 @@
 
 void ts_language_table_entry(const TSLanguage *self, TSStateId state,
                              TSSymbol symbol, TableEntry *result) {
-  if (symbol == ts_builtin_sym_error) {
+  if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) {
     result->action_count = 0;
     result->is_reusable = false;
     result->actions = NULL;
@@ -27,8 +27,10 @@ uint32_t ts_language_version(const TSLanguage *language) {
 }
 
 TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *language, TSSymbol symbol) {
-  if (symbol == ts_builtin_sym_error) {
+  if (symbol == ts_builtin_sym_error)  {
     return (TSSymbolMetadata){.visible = true, .named = true};
+  } else if (symbol == ts_builtin_sym_error_repeat) {
+    return (TSSymbolMetadata){.visible = false, .named = false};
   } else {
     return language->symbol_metadata[symbol];
   }
@@ -37,6 +39,8 @@ TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *language, TSSymbo
 const char *ts_language_symbol_name(const TSLanguage *language, TSSymbol symbol) {
   if (symbol == ts_builtin_sym_error) {
     return "ERROR";
+  } else if (symbol == ts_builtin_sym_error_repeat) {
+    return "_ERROR";
   } else {
     return language->symbol_names[symbol];
   }

src/runtime/language.h

@@ -8,6 +8,8 @@ extern "C" {
 #include "tree_sitter/parser.h"
 #include "runtime/tree.h"
 
+#define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1)
+
 typedef struct {
   const TSParseAction *actions;
   uint32_t action_count;
@@ -51,7 +53,7 @@ static inline bool ts_language_has_reduce_action(const TSLanguage *self,
 static inline TSStateId ts_language_next_state(const TSLanguage *self,
                                                TSStateId state,
                                                TSSymbol symbol) {
-  if (symbol == ts_builtin_sym_error) {
+  if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) {
     return 0;
   } else if (symbol < self->token_count) {
     uint32_t count;

src/runtime/parser.c

@@ -639,6 +639,20 @@ static StackSliceArray parser__reduce(Parser *self, StackVersion version, TSSymb
     for (uint32_t j = parent->children.size; j < slice.trees.size; j++) {
       ts_stack_push(self->stack, slice.version, slice.trees.contents[j], false, next_state);
     }
+
+    if (ts_stack_version_count(self->stack) > MAX_VERSION_COUNT) {
+      i++;
+      while (i < pop.size) {
+        StackSlice slice = pop.contents[i];
+        ts_tree_array_delete(&self->tree_pool, &slice.trees);
+        ts_stack_halt(self->stack, slice.version);
+        i++;
+      }
+      while (ts_stack_version_count(self->stack) > slice.version + 1) {
+        ts_stack_remove_version(self->stack, slice.version + 1);
+      }
+      break;
+    }
   }
 
   for (StackVersion i = initial_version_count; i < ts_stack_version_count(self->stack); i++) {
@@ -720,12 +734,23 @@ static void parser__accept(Parser *self, StackVersion version, Tree *lookahead)
 
 static bool parser__do_all_potential_reductions(Parser *self, StackVersion starting_version,
                                                 TSSymbol lookahead_symbol) {
-  bool result = false;
-  for (StackVersion version = starting_version;
-       ts_stack_version_count(self->stack) < MAX_VERSION_COUNT;) {
+  uint32_t initial_version_count = ts_stack_version_count(self->stack);
+
+  bool can_shift_lookahead_symbol = false;
+  StackVersion version = starting_version;
+  for (unsigned i = 0; true; i++) {
     uint32_t version_count = ts_stack_version_count(self->stack);
     if (version >= version_count) break;
 
+    bool merged = false;
+    for (StackVersion i = initial_version_count; i < version; i++) {
+      if (ts_stack_merge(self->stack, i, version)) {
+        merged = true;
+        break;
+      }
+    }
+    if (merged) continue;
+
     TSStateId state = ts_stack_state(self->stack, version);
     bool has_shift_action = false;
     array_clear(&self->reduce_actions);
@@ -747,7 +772,7 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start
         switch (action.type) {
           case TSParseActionTypeShift:
           case TSParseActionTypeRecover:
-            if (!action.params.extra) has_shift_action = true;
+            if (!action.params.extra && !action.params.repetition) has_shift_action = true;
             break;
           case TSParseActionTypeReduce:
             if (action.params.child_count > 0)
@@ -763,9 +788,9 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start
       }
     }
 
-    bool has_reduce_action = self->reduce_actions.size > 0;
     for (uint32_t i = 0; i < self->reduce_actions.size; i++) {
       ReduceAction action = self->reduce_actions.contents[i];
+
       parser__reduce(
         self, version, action.symbol, action.count,
         action.dynamic_precedence, action.alias_sequence_id,
@@ -774,14 +799,12 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start
     }
 
     if (has_shift_action) {
-      result = true;
-    } else {
-      if (has_reduce_action) {
-        ts_stack_renumber_version(self->stack, version_count, version);
-        continue;
-      } else if (lookahead_symbol != 0) {
-        ts_stack_remove_version(self->stack, version);
-      }
+      can_shift_lookahead_symbol = true;
+    } else if (self->reduce_actions.size > 0 && i < MAX_VERSION_COUNT) {
+      ts_stack_renumber_version(self->stack, version_count, version);
+      continue;
+    } else if (lookahead_symbol != 0) {
+      ts_stack_remove_version(self->stack, version);
     }
 
     if (version == starting_version) {
@@ -790,7 +813,8 @@ static bool parser__do_all_potential_reductions(Parser *self, StackVersion start
       version++;
     }
   }
-  return result;
+
+  return can_shift_lookahead_symbol;
 }
 
 static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lookahead_symbol) {
@@ -830,7 +854,11 @@ static void parser__handle_error(Parser *self, StackVersion version, TSSymbol lo
             self, version_with_missing_tree,
             lookahead_symbol
           )) {
-            LOG("recover_with_missing symbol:%s, state:%u", SYM_NAME(missing_symbol), state_after_missing_symbol);
+            LOG(
+              "recover_with_missing symbol:%s, state:%u",
+              SYM_NAME(missing_symbol),
+              ts_stack_state(self->stack, version_with_missing_tree)
+            );
             did_insert_missing_token = true;
             break;
           }
@@ -894,13 +922,14 @@ static bool parser__recover_to_state(Parser *self, StackVersion version, unsigne
       continue;
     }
 
-    StackSliceArray error_pop = ts_stack_pop_error(self->stack, slice.version);
-    if (error_pop.size > 0) {
-      StackSlice error_slice = error_pop.contents[0];
-      array_push_all(&error_slice.trees, &slice.trees);
-      array_delete(&slice.trees);
-      slice.trees = error_slice.trees;
-      ts_stack_renumber_version(self->stack, error_slice.version, slice.version);
+    TreeArray error_trees = ts_stack_pop_error(self->stack, slice.version);
+    if (error_trees.size > 0) {
+      assert(error_trees.size == 1);
+      array_splice(&slice.trees, 0, 0, &error_trees.contents[0]->children);
+      for (unsigned j = 0; j < error_trees.contents[0]->children.size; j++) {
+        ts_tree_retain(slice.trees.contents[j]);
+      }
+      ts_tree_array_delete(&self->tree_pool, &error_trees);
     }
 
     TreeArray trailing_extras = ts_tree_array_remove_trailing_extras(&slice.trees);
@@ -930,41 +959,51 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead)
   unsigned previous_version_count = ts_stack_version_count(self->stack);
   Length position = ts_stack_position(self->stack, version);
   StackSummary *summary = ts_stack_get_summary(self->stack, version);
-  unsigned depth_since_error = ts_stack_node_count_since_error(self->stack, version);
+  unsigned node_count_since_error = ts_stack_node_count_since_error(self->stack, version);
+  unsigned current_error_cost = ts_stack_error_cost(self->stack, version);
 
-  for (unsigned i = 0; i < summary->size; i++) {
-    StackSummaryEntry entry = summary->contents[i];
+  if (summary && lookahead->symbol != ts_builtin_sym_error) {
+    for (unsigned i = 0; i < summary->size; i++) {
+      StackSummaryEntry entry = summary->contents[i];
 
-    if (entry.state == ERROR_STATE) continue;
-    if (entry.position.bytes == position.bytes) continue;
-    unsigned depth = entry.depth + depth_since_error;
-    if (depth > MAX_SUMMARY_DEPTH) break;
+      if (entry.state == ERROR_STATE) continue;
+      if (entry.position.bytes == position.bytes) continue;
+      unsigned depth = entry.depth;
+      if (node_count_since_error > 0) depth++;
 
-    unsigned new_cost =
-      depth * ERROR_COST_PER_SKIPPED_TREE +
-      (position.bytes - entry.position.bytes) * ERROR_COST_PER_SKIPPED_CHAR +
-      (position.extent.row - entry.position.extent.row) * ERROR_COST_PER_SKIPPED_LINE;
-    if (parser__better_version_exists(self, version, false, new_cost)) break;
+      bool would_merge = false;
+      for (unsigned j = 0; j < previous_version_count; j++) {
+        if (
+          ts_stack_state(self->stack, j) == entry.state &&
+          ts_stack_position(self->stack, j).bytes == position.bytes
+        ) {
+          would_merge = true;
+          break;
+        }
+      }
 
-    if (ts_language_has_actions(self->language, entry.state, lookahead->symbol)) {
-      if (parser__recover_to_state(self, version, depth, entry.state)) {
-        did_recover = true;
-        LOG("recover state:%u, depth:%u", entry.state, depth);
-        LOG_STACK();
-        break;
+      if (would_merge) continue;
+
+      unsigned new_cost =
+        current_error_cost +
+        entry.depth * ERROR_COST_PER_SKIPPED_TREE +
+        (position.bytes - entry.position.bytes) * ERROR_COST_PER_SKIPPED_CHAR +
+        (position.extent.row - entry.position.extent.row) * ERROR_COST_PER_SKIPPED_LINE;
+      if (parser__better_version_exists(self, version, false, new_cost)) break;
+
+      if (ts_language_has_actions(self->language, entry.state, lookahead->symbol)) {
+        if (parser__recover_to_state(self, version, depth, entry.state)) {
+          did_recover = true;
+          LOG("recover_to_previous state:%u, depth:%u", entry.state, depth);
+          LOG_STACK();
+          break;
+        }
       }
     }
   }
 
   for (unsigned i = previous_version_count; i < ts_stack_version_count(self->stack); i++) {
-    if (ts_stack_is_active(self->stack, i)) {
-      for (unsigned j = 0; j < i; j++) {
-        if (ts_stack_can_merge(self->stack, j, i)) {
-          ts_stack_remove_version(self->stack, i--);
-          break;
-        }
-      }
-    } else {
+    if (!ts_stack_is_active(self->stack, i)) {
       ts_stack_remove_version(self->stack, i--);
     }
   }
@@ -983,15 +1022,56 @@ static void parser__recover(Parser *self, StackVersion version, Tree *lookahead)
     return;
   }
 
+  unsigned new_cost =
+    current_error_cost + ERROR_COST_PER_SKIPPED_TREE +
+    ts_tree_total_bytes(lookahead) * ERROR_COST_PER_SKIPPED_CHAR +
+    ts_tree_total_size(lookahead).extent.row * ERROR_COST_PER_SKIPPED_LINE;
+
+  if (parser__better_version_exists(self, version, false, new_cost)) {
+    ts_stack_halt(self->stack, version);
+    return;
+  }
+
   unsigned n;
   const TSParseAction *actions = ts_language_actions(self->language, 1, lookahead->symbol, &n);
-  bool extra = n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.extra;
-  parser__shift(self, version, ERROR_STATE, lookahead, extra);
+  if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.extra) {
+    lookahead->extra = true;
+  }
 
-  if (parser__better_version_exists(self, version, true, ts_stack_error_cost(self->stack, version))) {
-    ts_stack_halt(self->stack, version);
-  } else {
-    LOG("skip_token symbol:%s", SYM_NAME(lookahead->symbol));
+  LOG("skip_token symbol:%s", SYM_NAME(lookahead->symbol));
+  ts_tree_retain(lookahead);
+  TreeArray children = array_new();
+  array_grow(&children, 1);
+  array_push(&children, lookahead);
+  Tree *error_repeat = ts_tree_make_node(
+    &self->tree_pool,
+    ts_builtin_sym_error_repeat,
+    &children,
+    0,
+    self->language
+  );
+
+  if (node_count_since_error > 0) {
+    StackSliceArray pop = ts_stack_pop_count(self->stack, version, 1);
+    assert(pop.size == 1);
+    assert(pop.contents[0].trees.size == 1);
+    ts_stack_renumber_version(self->stack, pop.contents[0].version, version);
+    array_push(&pop.contents[0].trees, error_repeat);
+    error_repeat = ts_tree_make_node(
+      &self->tree_pool,
+      ts_builtin_sym_error_repeat,
+      &pop.contents[0].trees,
+      0,
+      self->language
+    );
+  }
+
+  ts_stack_push(self->stack, version, error_repeat, false, ERROR_STATE);
+
+  if (lookahead->has_external_tokens) {
+    ts_stack_set_last_external_token(
+      self->stack, version, ts_tree_last_external_token(lookahead)
+    );
   }
 }
 
@@ -1011,6 +1091,10 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re
           if (action.params.repetition) break;
           TSStateId next_state;
           if (action.params.extra) {
+
+            // TODO remove when TREE_SITTER_LANGUAGE_VERSION 9 is out.
+            if (state == ERROR_STATE) continue;
+
             next_state = state;
             LOG("shift_extra");
           } else {
@@ -1065,7 +1149,8 @@ static void parser__advance(Parser *self, StackVersion version, ReusableNode *re
       ts_stack_renumber_version(self->stack, last_reduction_version, version);
       LOG_STACK();
     } else if (state == ERROR_STATE) {
-      ts_stack_push(self->stack, version, lookahead, false, ERROR_STATE);
+      parser__recover(self, version, lookahead);
+      ts_tree_release(&self->tree_pool, lookahead);
       return;
     } else if (!parser__breakdown_top_of_stack(self, version)) {
       LOG("detect_error");

src/runtime/stack.c

@@ -142,21 +142,7 @@ static StackNode *stack_node_new(StackNode *previous_node, Tree *tree, bool is_p
       node->error_cost += tree->error_cost;
       node->position = length_add(node->position, ts_tree_total_size(tree));
       node->dynamic_precedence += tree->dynamic_precedence;
-      if (!tree->extra) {
-        node->node_count += tree->node_count;
-
-        if (state == ERROR_STATE) {
-          node->error_cost +=
-            ERROR_COST_PER_SKIPPED_TREE * ((tree->visible || tree->children.size == 0) ? 1 : tree->visible_child_count) +
-            ERROR_COST_PER_SKIPPED_CHAR * tree->size.bytes +
-            ERROR_COST_PER_SKIPPED_LINE * tree->size.extent.row;
-          if (previous_node->links[0].tree) {
-            node->error_cost +=
-              ERROR_COST_PER_SKIPPED_CHAR * tree->padding.bytes +
-              ERROR_COST_PER_SKIPPED_LINE * tree->padding.extent.row;
-          }
-        }
-      }
+      if (!tree->extra) node->node_count += tree->node_count;
     }
   } else {
     node->position = length_zero();
@@ -400,7 +386,9 @@ void ts_stack_set_last_external_token(Stack *self, StackVersion version, Tree *t
 unsigned ts_stack_error_cost(const Stack *self, StackVersion version) {
   StackHead *head = array_get(&self->heads, version);
   unsigned result = head->node->error_cost;
-  if (head->node->state == ERROR_STATE || head->status == StackStatusPaused) {
+  if (
+    head->status == StackStatusPaused ||
+    (head->node->state == ERROR_STATE && !head->node->links[0].tree)) {
     result += ERROR_COST_PER_RECOVERY;
   }
   return result;
@@ -408,6 +396,9 @@ unsigned ts_stack_error_cost(const Stack *self, StackVersion version) {
 
 unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version) {
   StackHead *head = array_get(&self->heads, version);
+  if (head->node->node_count < head->node_count_at_last_error) {
+    head->node_count_at_last_error = head->node->node_count;
+  }
   return head->node->node_count - head->node_count_at_last_error;
 }
 
@@ -482,15 +473,21 @@ inline StackAction pop_error_callback(void *payload, const Iterator *iterator) {
   }
 }
 
-StackSliceArray ts_stack_pop_error(Stack *self, StackVersion version) {
+TreeArray ts_stack_pop_error(Stack *self, StackVersion version) {
   StackNode *node = array_get(&self->heads, version)->node;
   for (unsigned i = 0; i < node->link_count; i++) {
     if (node->links[i].tree && node->links[i].tree->symbol == ts_builtin_sym_error) {
       bool found_error = false;
-      return stack__iter(self, version, pop_error_callback, &found_error, true);
+      StackSliceArray pop = stack__iter(self, version, pop_error_callback, &found_error, true);
+      if (pop.size > 0) {
+        assert(pop.size == 1);
+        ts_stack_renumber_version(self, pop.contents[0].version, version);
+        return pop.contents[0].trees;
+      }
+      break;
     }
   }
-  return (StackSliceArray){.size = 0};
+  return (TreeArray){.size = 0};
 }
 
 inline StackAction pop_all_callback(void *payload, const Iterator *iterator) {
@@ -550,8 +547,14 @@ void ts_stack_remove_version(Stack *self, StackVersion version) {
 void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) {
   assert(v2 < v1);
   assert((uint32_t)v1 < self->heads.size);
-  stack_head_delete(&self->heads.contents[v2], &self->node_pool, self->tree_pool);
-  self->heads.contents[v2] = self->heads.contents[v1];
+  StackHead *source_head = &self->heads.contents[v1];
+  StackHead *target_head = &self->heads.contents[v2];
+  if (target_head->summary && !source_head->summary) {
+    source_head->summary = target_head->summary;
+    target_head->summary = NULL;
+  }
+  stack_head_delete(target_head, &self->node_pool, self->tree_pool);
+  *target_head = *source_head;
   array_erase(&self->heads, v1);
 }
 
@@ -578,8 +581,8 @@ bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2) {
   for (uint32_t i = 0; i < head2->node->link_count; i++) {
     stack_node_add_link(head1->node, head2->node->links[i]);
   }
-  if (head2->node_count_at_last_error > head1->node_count_at_last_error) {
-    head1->node_count_at_last_error = head2->node_count_at_last_error;
+  if (head1->node->state == ERROR_STATE) {
+    head1->node_count_at_last_error = head1->node->node_count;
   }
   ts_stack_remove_version(self, version2);
   return true;
@@ -593,6 +596,7 @@ bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version
     head2->status == StackStatusActive &&
     head1->node->state == head2->node->state &&
     head1->node->position.bytes == head2->node->position.bytes &&
+    head1->node->error_cost == head2->node->error_cost &&
     ts_tree_external_token_state_eq(head1->last_external_token, head2->last_external_token);
 }
 

src/runtime/stack.h

@@ -65,7 +65,7 @@ void ts_stack_push(Stack *, StackVersion, Tree *, bool, TSStateId);
 StackSliceArray ts_stack_pop_count(Stack *, StackVersion, uint32_t count);
 
 // Remove an error at the top of the given version of the stack.
-StackSliceArray ts_stack_pop_error(Stack *, StackVersion);
+TreeArray ts_stack_pop_error(Stack *, StackVersion);
 
 // Remove any pending trees from the top of the given version of the stack.
 StackSliceArray ts_stack_pop_pending(Stack *, StackVersion);

src/runtime/tree.c

@@ -324,7 +324,9 @@ void ts_tree_set_children(Tree *self, TreeArray *children, const TSLanguage *lan
       self->size = length_add(self->size, ts_tree_total_size(child));
     }
 
-    self->error_cost += child->error_cost;
+    if (child->symbol != ts_builtin_sym_error_repeat) {
+      self->error_cost += child->error_cost;
+    }
     self->dynamic_precedence += child->dynamic_precedence;
     self->node_count += child->node_count;
 
@@ -351,13 +353,18 @@ void ts_tree_set_children(Tree *self, TreeArray *children, const TSLanguage *lan
     if (!child->extra) non_extra_index++;
   }
 
-  if (self->symbol == ts_builtin_sym_error) {
+  if (self->symbol == ts_builtin_sym_error || self->symbol == ts_builtin_sym_error_repeat) {
     self->error_cost += ERROR_COST_PER_RECOVERY +
                         ERROR_COST_PER_SKIPPED_CHAR * self->size.bytes +
                         ERROR_COST_PER_SKIPPED_LINE * self->size.extent.row;
     for (uint32_t i = 0; i < self->children.size; i++) {
-      if (!self->children.contents[i]->extra) {
+      Tree *child = self->children.contents[i];
+      if (child->extra) continue;
+      if (child->symbol == ts_builtin_sym_error && child->children.size == 0) continue;
+      if (child->visible) {
         self->error_cost += ERROR_COST_PER_SKIPPED_TREE;
+      } else {
+        self->error_cost += ERROR_COST_PER_SKIPPED_TREE * child->visible_child_count;
       }
     }
   }
@@ -387,26 +394,16 @@ Tree *ts_tree_make_node(TreePool *pool, TSSymbol symbol, TreeArray *children,
                         unsigned alias_sequence_id, const TSLanguage *language) {
   Tree *result = ts_tree_make_leaf(pool, symbol, length_zero(), length_zero(), language);
   result->alias_sequence_id = alias_sequence_id;
+  if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) {
+    result->fragile_left = true;
+    result->fragile_right = true;
+  }
   ts_tree_set_children(result, children, language);
   return result;
 }
 
 Tree *ts_tree_make_error_node(TreePool *pool, TreeArray *children, const TSLanguage *language) {
-  for (uint32_t i = 0; i < children->size; i++) {
-    Tree *child = children->contents[i];
-    if (child->symbol == ts_builtin_sym_error && child->children.size > 0) {
-      array_splice(children, i, 1, &child->children);
-      i += child->children.size - 1;
-      for (uint32_t j = 0; j < child->children.size; j++)
-        ts_tree_retain(child->children.contents[j]);
-      ts_tree_release(pool, child);
-    }
-  }
-
-  Tree *result = ts_tree_make_node(pool, ts_builtin_sym_error, children, 0, language);
-  result->fragile_left = true;
-  result->fragile_right = true;
-  return result;
+  return ts_tree_make_node(pool, ts_builtin_sym_error, children, 0, language);
 }
 
 Tree *ts_tree_make_missing_leaf(TreePool *pool, TSSymbol symbol, const TSLanguage *language) {

test/fixtures/error_corpus/c_errors.txt

@@ -81,7 +81,9 @@ int main() {
     (function_declarator (identifier) (parameter_list))
     (compound_statement
       (if_statement
-        (field_expression (identifier) (MISSING))
+        (field_expression
+          (identifier)
+          (MISSING))
         (compound_statement
           (expression_statement (call_expression (identifier) (argument_list)))
           (expression_statement (call_expression (identifier) (argument_list)))

test/fixtures/error_corpus/javascript_errors.txt

@@ -77,15 +77,12 @@ if ({a: 'b'} {c: 'd'}) {
       (ERROR (object (pair (property_identifier) (string))))
       (object (pair (property_identifier) (string))))
     (statement_block
-      (expression_statement (assignment_expression
-        (identifier)
-        (call_expression
-          (function (formal_parameters (identifier)) (statement_block (expression_statement (identifier))))
-          (ERROR)
-          (arguments (identifier))))
-          (MISSING))
-      (statement_block
-        (expression_statement (identifier))))))
+      (expression_statement
+        (assignment_expression
+          (identifier)
+          (function (formal_parameters (identifier)) (statement_block (expression_statement (identifier)))))
+        (MISSING))
+      (function (formal_parameters (identifier)) (statement_block (expression_statement (identifier)))))))
 
 ===================================================
 Extra tokens at the end of the file
@@ -150,3 +147,33 @@ const a = `b c ${d +} f g`
     (variable_declarator
       (identifier)
       (template_string (template_substitution (identifier) (ERROR))))))
+
+=========================================================
+Long sequences of invalid tokens
+=========================================================
+
+function main(x) {
+  console.log('a');
+  what??????????????????????????????????????????????????
+  console.log('b');
+  return {};
+}
+
+---
+
+(program
+  (function
+    (identifier)
+    (formal_parameters (identifier))
+    (statement_block
+      (expression_statement
+        (call_expression
+          (member_expression (identifier) (property_identifier))
+          (arguments (string))))
+      (expression_statement
+        (identifier)
+        (ERROR
+          (call_expression
+            (member_expression (identifier) (property_identifier))
+            (arguments (string)))))
+      (return_statement (object)))))