Fix error in document editing

When breaking down the stack in parser.c, the previous code
would not account for ubiquitous tokens. This was a problem
for a long time, but wasn't noticed until ubiquitous tokens
started being used to represent separator characters

spec/runtime/document_spec.cc

@@ -63,13 +63,16 @@ describe("Document", [&]() {
     });
   });
 
-  describe_skip("edit", [&]() {
+  describe("edit", [&]() {
     SpyReader *reader;
 
     before_each([&]() {
       reader = new SpyReader("{ \"key\": [1, 2] }", 5);
       ts_document_set_language(doc, ts_language_json());
       ts_document_set_input(doc, reader->input);
+
+      AssertThat(ts_node_string(ts_document_root_node(doc)), Equals(
+          "(DOCUMENT (object (string) (array (number) (number))))"));
     });
 
     after_each([&]() {
@@ -87,7 +90,7 @@ describe("Document", [&]() {
 
       it("updates the parse tree", [&]() {
         AssertThat(string(ts_node_string(ts_document_root_node(doc))), Equals(
-            "(DOCUMENT (object (string) (array (number) (number)) (string) (number)))"));
+            "(DOCUMENT (object (string) (array (number) (number)) (string) (value (number))))"));
       });
 
       it("re-reads only the changed portion of the input", [&]() {

src/compiler/build_tables/build_lex_table.cc

@@ -38,11 +38,9 @@ class LexTableBuilder {
       if (symbol == rules::ERROR())
         continue;
       else if (symbol == rules::END_OF_INPUT())
-        result.insert(LexItem(
-            symbol, CharacterSet().include(0).copy()));
+        result.insert(LexItem(symbol, CharacterSet().include(0).copy()));
       else if (symbol.is_token())
-        result.insert(
-            LexItem(symbol, lex_grammar.rule(symbol)));
+        result.insert(LexItem(symbol, lex_grammar.rule(symbol)));
     }
     return result;
   }

src/compiler/grammar.cc

@@ -12,8 +12,7 @@ using rules::rule_ptr;
 
 Grammar::Grammar(
     const std::vector<std::pair<std::string, rules::rule_ptr> > &rules)
-    : rules_(rules),
-      ubiquitous_tokens_({}) {}
+    : rules_(rules), ubiquitous_tokens_({}) {}
 
 bool Grammar::operator==(const Grammar &other) const {
   if (other.rules_.size() != rules_.size())

src/runtime/parser.c

@@ -38,14 +38,22 @@ static size_t breakdown_stack(TSParser *parser, TSInputEdit *edit) {
     stack->size--;
     position -= node->size;
 
+    DEBUG_PARSE("BREAKDOWN %s %u", parser->language->symbol_names[node->symbol],
+                ts_stack_top_state(stack));
+
     for (size_t i = 0; i < child_count && position < edit->position; i++) {
       TSTree *child = children[i];
       TSStateId state = ts_stack_top_state(stack);
-      TSStateId next_state =
-          action_for(parser->language, state, child->symbol).data.to_state;
+      TSParseAction action = action_for(parser->language, state, child->symbol);
+      TSStateId next_state = (action.type == TSParseActionTypeShift)
+                                 ? action.data.to_state
+                                 : state;
       ts_stack_push(stack, next_state, child);
       ts_tree_retain(child);
       position += child->size;
+
+      DEBUG_PARSE("PUT_BACK %s %u",
+                  parser->language->symbol_names[child->symbol], next_state);
     }
 
     ts_tree_release(node);
@@ -71,9 +79,6 @@ static void reduce(TSParser *parser, TSSymbol symbol, size_t child_count) {
   TSStack *stack = &parser->stack;
   parser->next_lookahead = parser->lookahead;
 
-  /* size_t trailing_extra_count; */
-  /* TSTree **trailing_extras = ts_stack_pop_extras(stack, &trailing_extra_count); */
-
   /*
    *  Walk down the stack to determine which symbols will be reduced.
    *  The child node count is known ahead of time, but some children
@@ -93,15 +98,6 @@ static void reduce(TSParser *parser, TSSymbol symbol, size_t child_count) {
   int hidden = parser->language->hidden_symbol_flags[symbol];
   parser->lookahead = ts_tree_make_node(symbol, child_count, children, hidden);
   ts_stack_shrink(stack, stack->size - child_count);
-
-  /* TSStateId state = ts_stack_top_state(stack); */
-  /* if (trailing_extras) { */
-    /* for (size_t i = 0; i < trailing_extra_count; i++) { */
-      /* ts_stack_push(&parser->stack, state, trailing_extras[i]); */
-      /* ts_tree_release(trailing_extras[i]); */
-    /* } */
-    /* free(trailing_extras); */
-  /* } */
 }
 
 static int reduce_extra(TSParser *parser, TSSymbol symbol) {
@@ -169,8 +165,8 @@ static int handle_error(TSParser *parser) {
           DEBUG_PARSE("RECOVER %u", state_after_error);
           ts_stack_shrink(&parser->stack, i + 1);
           error->size = ts_lexer_position(&parser->lexer) -
-              parser->lookahead->size -
-              ts_stack_right_position(&parser->stack);
+                        parser->lookahead->size -
+                        ts_stack_right_position(&parser->stack);
           ts_stack_push(&parser->stack, state_after_error, error);
           ts_tree_release(error);
           return 1;

src/runtime/stack.h

@@ -29,7 +29,7 @@ TSTree **ts_stack_pop_extras(TSStack *, size_t *);
 
 #define TS_STACK_FROM_TOP(stack, entry, index)               \
   size_t index = stack.size - 1;                             \
-  (void)index;                                               \
+  (void) index;                                              \
   for (TSStackEntry *entry = stack.entries + stack.size - 1; \
        entry >= stack.entries; entry-- && index--)