From 10ab7032a6c99a39f7cb0e4dc39ef0624b7a574b Mon Sep 17 00:00:00 2001
From: Max Brunsfeld <maxbrunsfeld@gmail.com>
Date: Sun, 11 Nov 2018 21:36:31 -0800
Subject: [PATCH] Fix incorrect node reuse for edits right at EOF

---
 src/runtime/parser.c              |  2 +-
 test/integration/real_grammars.cc | 31 +++++++++++++++++++++++++++----
 test/runtime/parser_test.cc       | 15 +++++++++++++++
 3 files changed, 43 insertions(+), 5 deletions(-)

diff --git a/src/runtime/parser.c b/src/runtime/parser.c
index 0bdefcbf..d87f25d4 100644
--- a/src/runtime/parser.c
+++ b/src/runtime/parser.c
@@ -416,7 +416,7 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId pa
     last_byte_scanned = self->lexer.current_position.bytes;
   }
 
-  if (self->lexer.data.lookahead != 0) last_byte_scanned++;
+  last_byte_scanned++;
 
   Subtree result;
   if (skipped_error) {
diff --git a/test/integration/real_grammars.cc b/test/integration/real_grammars.cc
index 556ddf44..8d0807ef 100644
--- a/test/integration/real_grammars.cc
+++ b/test/integration/real_grammars.cc
@@ -11,6 +11,32 @@
 #include "helpers/tree_helpers.h"
 #include <set>
 
+TSInputEdit do_random_edit(Generator &random, SpyInput &input, TSTree *tree) {
+  size_t choice = random(10);
+
+  if (choice < 2) {
+    // Insert text at end
+    string inserted_text = random.words(1);
+    return input.replace(input.content.size(), 0, inserted_text);
+  } else if (choice < 5) {
+    // Delete text from end
+    size_t deletion_size = random(10);
+    if (deletion_size > input.content.size()) deletion_size = input.content.size();
+    return input.replace(input.content.size() - deletion_size, deletion_size, "");
+  } else if (choice < 8) {
+    // Insert at random position
+    size_t position = random(input.content.size() + 1);
+    string inserted_text = random.words(1 + random(3));
+    return input.replace(position, 0, inserted_text);
+  } else {
+    // Replace at random position
+    size_t position = random(input.content.size() + 1);
+    size_t deletion_size = random(input.content.size() + 1 - position);
+    string inserted_text = random.words(1 + random(4));
+    return input.replace(position, deletion_size, inserted_text);
+  }
+}
+
 START_TEST;
 
 if (TREE_SITTER_SEED == -1) return;
@@ -79,10 +105,7 @@ for (auto &language_name : test_languages) {
 
           // Perform a random series of edits.
           for (unsigned j = 0; j < edit_count; j++) {
-            size_t edit_position = random(input.content.size());
-            size_t deletion_size = random(input.content.size() - edit_position);
-            string inserted_text = random.words(1 + random(4));
-            TSInputEdit edit = input.replace(edit_position, deletion_size, inserted_text);
+            TSInputEdit edit = do_random_edit(random, input, tree);
             ts_tree_edit(tree, &edit);
             if (debug_graphs_enabled) {
               ts_tree_print_dot_graph(tree, stderr);
diff --git a/test/runtime/parser_test.cc b/test/runtime/parser_test.cc
index cb168234..c74e5b34 100644
--- a/test/runtime/parser_test.cc
+++ b/test/runtime/parser_test.cc
@@ -452,6 +452,21 @@ describe("Parser", [&]() {
       });
     });
 
+    describe("insertions at the end of the file", [&]() {
+      it("doesn't incorrectly reuse nodes at EOF", [&]() {
+        ts_parser_set_language(parser, load_real_language("javascript"));
+
+        set_text("ab");
+        assert_root_node("(program (expression_statement (identifier)))");
+
+        insert_text(input->content.size(), " ");
+        insert_text(input->content.size(), "+=");
+        insert_text(input->content.size(), " ");
+        insert_text(input->content.size(), "12");
+        assert_root_node("(program (expression_statement (augmented_assignment_expression (identifier) (number))))");
+      });
+    });
+
     it("does not try to reuse nodes that are within the edited region", [&]() {
       ts_parser_set_language(parser, load_real_language("javascript"));
       set_text("{ x: (b.c) };");