Requery the parse table when breaking down the parse stack on invalid lookahead (#636)

* Requery parse table after breaking down parse stack due to invalid lookahead

* Include Ruby parser in randomized test suite

Ruby and PHP are our only two languages that use non-terminal extras.
Adding Ruby uncovered some bugs.

* Print edited source code when running parse --edit w/ debug flag

* Recompute lookahead when breaking down stack on invalid lookahead

* Fix stack summary leak when there are two discontinuities on a stack version

cli/src/parse.rs

@@ -70,10 +70,18 @@ pub fn parse_file_at_path(
     let mut stdout = stdout.lock();
 
     if let Some(mut tree) = tree {
-        for edit in edits {
+        if debug_graph && !edits.is_empty() {
+            println!("BEFORE:\n{}", String::from_utf8_lossy(&source_code));
+        }
+
+        for (i, edit) in edits.iter().enumerate() {
             let edit = parse_edit_flag(&source_code, edit)?;
             perform_edit(&mut tree, &mut source_code, &edit);
             tree = parser.parse(&source_code, Some(&tree)).unwrap();
+
+            if debug_graph {
+                println!("AFTER {}:\n{}", i, String::from_utf8_lossy(&source_code));
+            }
         }
 
         let duration = time.elapsed();

cli/src/tests/corpus_test.rs

@@ -24,6 +24,7 @@ const LANGUAGES: &'static [&'static str] = &[
     "json",
     "php",
     "python",
+    "ruby",
     "rust",
 ];
 

lib/src/parser.c

@@ -1339,6 +1339,7 @@ static bool ts_parser__advance(
     );
   }
 
+lex:
   // Otherwise, re-run the lexer.
   if (!lookahead.ptr) {
     lookahead = ts_parser__lex(self, version, state);
@@ -1500,6 +1501,10 @@ static bool ts_parser__advance(
     // push each of its children. Then try again to process the current
     // lookahead.
     if (ts_parser__breakdown_top_of_stack(self, version)) {
+      state = ts_stack_state(self->stack, version);
+      ts_subtree_release(&self->tree_pool, lookahead);
+      lookahead = NULL_SUBTREE;
+      goto lex;
       continue;
     }
 

lib/src/stack.c

@@ -571,7 +571,12 @@ void ts_stack_record_summary(Stack *self, StackVersion version, unsigned max_dep
   };
   array_init(session.summary);
   stack__iter(self, version, summarize_stack_callback, &session, -1);
-  self->heads.contents[version].summary = session.summary;
+  StackHead *head = &self->heads.contents[version];
+  if (head->summary) {
+    array_delete(head->summary);
+    ts_free(head->summary);
+  }
+  head->summary = session.summary;
 }
 
 StackSummary *ts_stack_get_summary(Stack *self, StackVersion version) {
@@ -743,6 +748,10 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f)
       ts_stack_error_cost(self, i)
     );
 
+    if (head->summary) {
+      fprintf(f, "\nsummary_size: %u", head->summary->size);
+    }
+
     if (head->last_external_token.ptr) {
       const ExternalScannerState *state = &head->last_external_token.ptr->external_scanner_state;
       const char *data = ts_external_scanner_state_data(state);