From b14951de9d84136ca4a9f10db3dd1e5778a462b8 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 15 Jun 2018 16:17:56 -0700 Subject: [PATCH 1/5] Included HTML parser in randomized test suite --- script/fetch-fixtures | 1 + test/integration/real_grammars.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/script/fetch-fixtures b/script/fetch-fixtures index 5b32a66e..89b9c5fa 100755 --- a/script/fetch-fixtures +++ b/script/fetch-fixtures @@ -30,3 +30,4 @@ fetch_grammar go master fetch_grammar ruby master fetch_grammar typescript master fetch_grammar bash master +fetch_grammar html master diff --git a/test/integration/real_grammars.cc b/test/integration/real_grammars.cc index 02752a34..0a0e6d7d 100644 --- a/test/integration/real_grammars.cc +++ b/test/integration/real_grammars.cc @@ -25,6 +25,7 @@ if (TREE_SITTER_SEED == -1) return; vector test_languages({ "javascript", "json", + "html", "c", "cpp", "python", From 75cf95bddcb3d2b7e4f8c393dd99d012f2855214 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 15 Jun 2018 16:18:08 -0700 Subject: [PATCH 2/5] Fix double free when an external token is copied --- src/runtime/subtree.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/runtime/subtree.c b/src/runtime/subtree.c index b2de81a1..69ee1cdc 100644 --- a/src/runtime/subtree.c +++ b/src/runtime/subtree.c @@ -35,6 +35,15 @@ void ts_external_scanner_state_init(ExternalScannerState *self, const char *data } } +ExternalScannerState ts_external_scanner_state_copy(const ExternalScannerState *self) { + ExternalScannerState result = *self; + if (self->length > sizeof(self->short_data)) { + result.long_data = ts_malloc(self->length); + memcpy(result.long_data, self->long_data, self->length); + } + return result; +} + void ts_external_scanner_state_delete(ExternalScannerState *self) { if (self->length > sizeof(self->short_data)) { ts_free(self->long_data); @@ -182,6 +191,8 @@ Subtree *ts_subtree_new_copy(SubtreePool *pool, const Subtree *self) { *result = *self; if (result->children.size > 0) { ts_subtree_array_copy(self->children, &result->children); + } else if (result->has_external_tokens) { + result->external_scanner_state = ts_external_scanner_state_copy(&self->external_scanner_state); } result->ref_count = 1; return result; From b29c5dbf15d583e6d098d6e9b8483ed854af68b4 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 15 Jun 2018 17:07:35 -0700 Subject: [PATCH 3/5] Rephrase ts_subtree_array_copy to avoid conspicuous null case This avoids a false positive error from the clang static analyzer, which doesn't understand the invariants of the Array type. --- src/runtime/subtree.c | 21 +++++++++------------ src/runtime/subtree.h | 2 +- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/runtime/subtree.c b/src/runtime/subtree.c index 69ee1cdc..89e9d0d5 100644 --- a/src/runtime/subtree.c +++ b/src/runtime/subtree.c @@ -67,20 +67,17 @@ bool ts_external_scanner_state_eq(const ExternalScannerState *a, const ExternalS // SubtreeArray -bool ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest) { - const Subtree **contents = NULL; - if (self.capacity > 0) { - contents = ts_calloc(self.capacity, sizeof(Subtree *)); - memcpy(contents, self.contents, self.size * sizeof(Subtree *)); - for (uint32_t i = 0; i < self.size; i++) { - ts_subtree_retain(contents[i]); - } - } - +void ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest) { dest->size = self.size; dest->capacity = self.capacity; - dest->contents = contents; - return true; + dest->contents = self.contents; + if (self.capacity > 0) { + dest->contents = ts_calloc(self.capacity, sizeof(Subtree *)); + memcpy(dest->contents, self.contents, self.size * sizeof(Subtree *)); + for (uint32_t i = 0; i < self.size; i++) { + ts_subtree_retain(dest->contents[i]); + } + } } void ts_subtree_array_delete(SubtreePool *pool, SubtreeArray *self) { diff --git a/src/runtime/subtree.h b/src/runtime/subtree.h index 66fc1f24..0fdba75f 100644 --- a/src/runtime/subtree.h +++ b/src/runtime/subtree.h @@ -79,7 +79,7 @@ typedef struct { void ts_external_scanner_state_init(ExternalScannerState *, const char *, unsigned); const char *ts_external_scanner_state_data(const ExternalScannerState *); -bool ts_subtree_array_copy(SubtreeArray, SubtreeArray *); +void ts_subtree_array_copy(SubtreeArray, SubtreeArray *); void ts_subtree_array_delete(SubtreePool *, SubtreeArray *); SubtreeArray ts_subtree_array_remove_trailing_extras(SubtreeArray *); void ts_subtree_array_reverse(SubtreeArray *); From 71f7bf2fcac98c32b3118f7dc425bc7e910a8c38 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 18 Jun 2018 10:06:59 -0700 Subject: [PATCH 4/5] Fetch HTML grammar as a test fixture on windows too --- script/fetch-fixtures.cmd | 1 + 1 file changed, 1 insertion(+) diff --git a/script/fetch-fixtures.cmd b/script/fetch-fixtures.cmd index 1effc141..187aba6d 100644 --- a/script/fetch-fixtures.cmd +++ b/script/fetch-fixtures.cmd @@ -9,6 +9,7 @@ call:fetch_grammar go master call:fetch_grammar ruby master call:fetch_grammar typescript master call:fetch_grammar bash master +call:fetch_grammar html master EXIT /B 0 :fetch_grammar From 703541e838bcfedb171f827f0041a0b7be5b84d7 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 18 Jun 2018 10:48:32 -0700 Subject: [PATCH 5/5] Explicitly initialize variable in ts_tree_cursor_goto_next_sibling --- src/runtime/tree_cursor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/tree_cursor.c b/src/runtime/tree_cursor.c index f2d9279f..73c08672 100644 --- a/src/runtime/tree_cursor.c +++ b/src/runtime/tree_cursor.c @@ -151,13 +151,13 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *_self) { uint32_t initial_size = self->stack.size; while (self->stack.size > 1) { - bool visible; TreeCursorEntry entry = array_pop(&self->stack); ChildIterator iterator = ts_tree_cursor_iterate_children(self); iterator.child_index = entry.child_index; iterator.structural_child_index = entry.structural_child_index; iterator.position = entry.position; + bool visible = false; ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible); if (visible && self->stack.size + 1 < initial_size) break;