From 343887c1ddc70e7d46fc9d64048b96daccb8a96f Mon Sep 17 00:00:00 2001
From: Max Brunsfeld <maxbrunsfeld@gmail.com>
Date: Mon, 6 Feb 2017 17:43:07 -0800
Subject: [PATCH] Fix miscounting of extra tokens when repairing errors

---
 .../error_corpus/javascript_errors.txt        | 18 +++++++++++++++
 src/runtime/parser.c                          | 10 +++-----
 src/runtime/tree.c                            | 23 +++++++++++++++++++
 src/runtime/tree.h                            |  1 +
 4 files changed, 45 insertions(+), 7 deletions(-)

diff --git a/spec/fixtures/error_corpus/javascript_errors.txt b/spec/fixtures/error_corpus/javascript_errors.txt
index 87ae22a8..8994aca2 100644
--- a/spec/fixtures/error_corpus/javascript_errors.txt
+++ b/spec/fixtures/error_corpus/javascript_errors.txt
@@ -76,6 +76,24 @@ a.b =
     (member_access (identifier) (identifier)))
   (ERROR))
 
+=================================================================
+An invalid token at the end of a construct with extra line breaks
+=================================================================
+
+a(
+  b,
+  c,,
+);
+
+---
+
+(program
+  (expression_statement
+    (function_call (identifier) (arguments
+      (identifier)
+      (identifier)
+      (ERROR)))))
+
 ===================================================
 Multi-line chained expressions in var declarations
 ===================================================
diff --git a/src/runtime/parser.c b/src/runtime/parser.c
index 191354a3..24e8281a 100644
--- a/src/runtime/parser.c
+++ b/src/runtime/parser.c
@@ -596,6 +596,7 @@ static inline const TSParseAction *parser__reductions_after_sequence(
     if (child_count == tree_count_below)
       break;
     Tree *tree = trees_below->contents[trees_below->size - 1 - i];
+    if (tree->extra) continue;
     TSStateId next_state = ts_language_next_state(self->language, state, tree->symbol);
     if (next_state == ERROR_STATE)
       return NULL;
@@ -607,6 +608,7 @@ static inline const TSParseAction *parser__reductions_after_sequence(
 
   for (uint32_t i = 0; i < trees_above->size; i++) {
     Tree *tree = trees_above->contents[i];
+    if (tree->extra) continue;
     TSStateId next_state = ts_language_next_state(self->language, state, tree->symbol);
     if (next_state == ERROR_STATE)
       return NULL;
@@ -738,7 +740,6 @@ static bool parser__repair_error(Parser *self, StackSlice slice,
   ReduceAction repair = session.best_repair;
   TSStateId next_state = session.best_repair_next_state;
   uint32_t skip_count = session.best_repair_skip_count;
-  uint32_t count_below = repair.count - session.tree_count_above_error;
   TSSymbol symbol = repair.symbol;
 
   StackSlice new_slice = array_pop(&pop.slices);
@@ -752,13 +753,8 @@ static bool parser__repair_error(Parser *self, StackSlice slice,
       ts_stack_remove_version(self->stack, other_slice.version);
   }
 
-  TreeArray skipped_children = array_new();
-  array_grow(&skipped_children, skip_count);
-  for (uint32_t i = count_below; i < children.size; i++)
-    array_push(&skipped_children, children.contents[i]);
-
+  TreeArray skipped_children = ts_tree_array_remove_last_n(&children, skip_count);
   Tree *error = ts_tree_make_error_node(&skipped_children);
-  children.size = count_below;
   array_push(&children, error);
 
   for (uint32_t i = 0; i < slice.trees.size; i++)
diff --git a/src/runtime/tree.c b/src/runtime/tree.c
index 858ad90e..0b10604e 100644
--- a/src/runtime/tree.c
+++ b/src/runtime/tree.c
@@ -61,6 +61,29 @@ uint32_t ts_tree_array_essential_count(const TreeArray *self) {
   return result;
 }
 
+TreeArray ts_tree_array_remove_last_n(TreeArray *self, uint32_t remove_count) {
+  TreeArray result = array_new();
+  if (self->size == 0 || remove_count == 0) return result;
+
+  uint32_t count = 0;
+  uint32_t split_index = self->size - 1;
+  for (; split_index + 1 > 0; split_index--) {
+    Tree *tree = self->contents[split_index];
+    if (!tree->extra) {
+      count++;
+      if (count == remove_count) break;
+    }
+  }
+
+  array_grow(&result, self->size - split_index);
+  for (uint32_t i = split_index; i < self->size; i++) {
+    array_push(&result, self->contents[i]);
+  }
+
+  self->size = split_index;
+  return result;
+}
+
 Tree *ts_tree_make_error(Length size, Length padding, char lookahead_char) {
   Tree *result = ts_tree_make_leaf(ts_builtin_sym_error, padding, size,
                                      (TSSymbolMetadata){
diff --git a/src/runtime/tree.h b/src/runtime/tree.h
index d5916e31..69ba8c38 100644
--- a/src/runtime/tree.h
+++ b/src/runtime/tree.h
@@ -68,6 +68,7 @@ typedef Array(TreePathEntry) TreePath;
 bool ts_tree_array_copy(TreeArray, TreeArray *);
 void ts_tree_array_delete(TreeArray *);
 uint32_t ts_tree_array_essential_count(const TreeArray *);
+TreeArray ts_tree_array_remove_last_n(TreeArray *, uint32_t);
 
 Tree *ts_tree_make_leaf(TSSymbol, Length, Length, TSSymbolMetadata);
 Tree *ts_tree_make_node(TSSymbol, uint32_t, Tree **, TSSymbolMetadata);