Add tests for editing all language examples

This drove out several fixes to the incremental parsing algorithm.
There are a few examples which still don't work and which have
been explicitly excluded, for now.

spec/runtime/languages/language_specs.cc

@@ -1,5 +1,8 @@
 #include "runtime/runtime_spec_helper.h"
 #include "runtime/helpers/read_test_entries.h"
+#include "runtime/helpers/spy_reader.h"
+#include "runtime/helpers/log_debugger.h"
+#include <set>
 
 extern "C" const TSLanguage *ts_language_javascript();
 extern "C" const TSLanguage *ts_language_json();
@@ -26,11 +29,40 @@ describe("Languages", [&]() {
       });
 
       for (auto &entry : test_entries_for_language(language_name)) {
-        it(("parses " + entry.description).c_str(), [&]() {
-          ts_document_set_input_string(doc, entry.input.c_str());
+        auto expect_the_correct_tree = [&]() {
           const char *node_string = ts_node_string(ts_document_root_node(doc));
           AssertThat(node_string, Equals(("(DOCUMENT " + entry.tree_string + ")").c_str()));
           free((void *)node_string);
+        };
+
+        it(("parses " + entry.description).c_str(), [&]() {
+          ts_document_set_input_string(doc, entry.input.c_str());
+          expect_the_correct_tree();
+        });
+
+        // TODO - make these tests pass
+        set<string> skipped({
+          "while loops",
+          "function expressions",
+        });
+
+        if (skipped.find(entry.description) != skipped.end())
+          continue;
+
+        it(("handles random insertions in " + entry.description).c_str(), [&]() {
+          SpyReader reader(entry.input, 3);
+          ts_document_set_input(doc, reader.input());
+
+          string garbage("%^&*");
+          size_t position = entry.input.size() / 2;
+
+          reader.insert(position, garbage);
+          ts_document_edit(doc, { position, garbage.size(), 0 });
+
+          reader.erase(position, garbage.size());
+          ts_document_edit(doc, { position, 0, garbage.size() });
+
+          expect_the_correct_tree();
         });
       }
     });

spec/runtime/parser_spec.cc

@@ -269,7 +269,7 @@ describe("Parser", [&]() {
         });
 
         it("re-reads only the changed portion of the input", [&]() {
-          AssertThat(reader->strings_read, Equals(vector<string>({ "123 + 5 ", "" })));
+          AssertThat(reader->strings_read, Equals(vector<string>({ "123 + 5 ", " 4", "" })));
         });
       });
 

src/runtime/parser.c

@@ -62,7 +62,7 @@ static TSLength break_down_left_stack(TSParser *parser, TSInputEdit edit) {
       TSStateId state = ts_stack_top_state(&parser->stack);
       TSParseAction action = get_action(parser->language, state, child->symbol);
       TSStateId next_state =
-          action.type == TSParseActionTypeShift ? action.data.to_state : state;
+          ts_tree_is_extra(child) ? state : action.data.to_state;
 
       DEBUG("push_left sym:%s state:%u", SYM_NAME(child->symbol), next_state);
       ts_stack_push(&parser->stack, next_state, child);
@@ -106,7 +106,8 @@ static TSTree *break_down_right_stack(TSParser *parser) {
 
     TSParseAction action = get_action(parser->language, state, node->symbol);
     bool is_usable = (action.type != TSParseActionTypeError) &&
-                     (node->symbol != ts_builtin_sym_error);
+                     (node->symbol != ts_builtin_sym_error) &&
+                     !ts_tree_is_extra(node);
     if (is_usable && right_subtree_start == current_position.chars) {
       ts_stack_shrink(&parser->right_stack, parser->right_stack.size - 1);
       return node;
@@ -147,6 +148,7 @@ static TSTree *get_next_node(TSParser *parser, TSStateId lex_state) {
 
     parser->lexer.lookahead = 0;
     parser->lexer.lookahead_size = 0;
+    parser->lexer.advance_fn(&parser->lexer, 0);
   } else {
     node = parser->language->lex_fn(&parser->lexer, lex_state);
   }