Merge pull request #177 from tree-sitter/html-in-randomized-tests

Include the HTML parser in the randomized test suite, fix discovered bug

script/fetch-fixtures

@@ -30,3 +30,4 @@ fetch_grammar go         master
 fetch_grammar ruby       master
 fetch_grammar typescript master
 fetch_grammar bash       master
+fetch_grammar html       master

script/fetch-fixtures.cmd

@@ -9,6 +9,7 @@ call:fetch_grammar go         master
 call:fetch_grammar ruby       master
 call:fetch_grammar typescript master
 call:fetch_grammar bash       master
+call:fetch_grammar html       master
 EXIT /B 0
 
 :fetch_grammar

src/runtime/subtree.c

@@ -35,6 +35,15 @@ void ts_external_scanner_state_init(ExternalScannerState *self, const char *data
   }
 }
 
+ExternalScannerState ts_external_scanner_state_copy(const ExternalScannerState *self) {
+  ExternalScannerState result = *self;
+  if (self->length > sizeof(self->short_data)) {
+    result.long_data = ts_malloc(self->length);
+    memcpy(result.long_data, self->long_data, self->length);
+  }
+  return result;
+}
+
 void ts_external_scanner_state_delete(ExternalScannerState *self) {
   if (self->length > sizeof(self->short_data)) {
     ts_free(self->long_data);
@@ -58,20 +67,17 @@ bool ts_external_scanner_state_eq(const ExternalScannerState *a, const ExternalS
 
 // SubtreeArray
 
-bool ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest) {
-  const Subtree **contents = NULL;
-  if (self.capacity > 0) {
-    contents = ts_calloc(self.capacity, sizeof(Subtree *));
-    memcpy(contents, self.contents, self.size * sizeof(Subtree *));
-    for (uint32_t i = 0; i < self.size; i++) {
-      ts_subtree_retain(contents[i]);
-    }
-  }
-
+void ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest) {
   dest->size = self.size;
   dest->capacity = self.capacity;
-  dest->contents = contents;
-  return true;
+  dest->contents = self.contents;
+  if (self.capacity > 0) {
+    dest->contents = ts_calloc(self.capacity, sizeof(Subtree *));
+    memcpy(dest->contents, self.contents, self.size * sizeof(Subtree *));
+    for (uint32_t i = 0; i < self.size; i++) {
+      ts_subtree_retain(dest->contents[i]);
+    }
+  }
 }
 
 void ts_subtree_array_delete(SubtreePool *pool, SubtreeArray *self) {
@@ -182,6 +188,8 @@ Subtree *ts_subtree_new_copy(SubtreePool *pool, const Subtree *self) {
   *result = *self;
   if (result->children.size > 0) {
     ts_subtree_array_copy(self->children, &result->children);
+  } else if (result->has_external_tokens) {
+    result->external_scanner_state = ts_external_scanner_state_copy(&self->external_scanner_state);
   }
   result->ref_count = 1;
   return result;

src/runtime/subtree.h

@@ -79,7 +79,7 @@ typedef struct {
 void ts_external_scanner_state_init(ExternalScannerState *, const char *, unsigned);
 const char *ts_external_scanner_state_data(const ExternalScannerState *);
 
-bool ts_subtree_array_copy(SubtreeArray, SubtreeArray *);
+void ts_subtree_array_copy(SubtreeArray, SubtreeArray *);
 void ts_subtree_array_delete(SubtreePool *, SubtreeArray *);
 SubtreeArray ts_subtree_array_remove_trailing_extras(SubtreeArray *);
 void ts_subtree_array_reverse(SubtreeArray *);

src/runtime/tree_cursor.c

@@ -151,13 +151,13 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *_self) {
   uint32_t initial_size = self->stack.size;
 
   while (self->stack.size > 1) {
-    bool visible;
     TreeCursorEntry entry = array_pop(&self->stack);
     ChildIterator iterator = ts_tree_cursor_iterate_children(self);
     iterator.child_index = entry.child_index;
     iterator.structural_child_index = entry.structural_child_index;
     iterator.position = entry.position;
 
+    bool visible = false;
     ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible);
     if (visible && self->stack.size + 1 < initial_size) break;
 

test/integration/real_grammars.cc

@@ -25,6 +25,7 @@ if (TREE_SITTER_SEED == -1) return;
 vector<string> test_languages({
   "javascript",
   "json",
+  "html",
   "c",
   "cpp",
   "python",