From 3dab0066bcb356616a205a6fb7472dfcb4a1b244 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 19 Sep 2018 10:52:21 -0700 Subject: [PATCH] Add missing padding bytes check before storing tree inline --- src/runtime/subtree.c | 1 + test/helpers/tree_helpers.cc | 26 ++++++++++++++++++++++---- test/helpers/tree_helpers.h | 2 +- test/integration/fuzzing-examples.cc | 3 +-- test/integration/real_grammars.cc | 20 +++++++------------- test/integration/test_grammars.cc | 4 +--- test/runtime/tree_test.cc | 15 ++++++++------- 7 files changed, 41 insertions(+), 30 deletions(-) diff --git a/src/runtime/subtree.c b/src/runtime/subtree.c index f0eb2361..72aa5d7c 100644 --- a/src/runtime/subtree.c +++ b/src/runtime/subtree.c @@ -160,6 +160,7 @@ static void ts_subtree_pool_free(SubtreePool *self, SubtreeHeapData *tree) { static inline bool ts_subtree_can_inline(Length padding, Length size) { return + padding.bytes < TS_MAX_INLINE_TREE_LENGTH && padding.extent.row < 16 && padding.extent.column < TS_MAX_INLINE_TREE_LENGTH && size.extent.row == 0 && diff --git a/test/helpers/tree_helpers.cc b/test/helpers/tree_helpers.cc index e8b3ace4..1d4c0d3a 100644 --- a/test/helpers/tree_helpers.cc +++ b/test/helpers/tree_helpers.cc @@ -4,6 +4,7 @@ #include using std::string; +using std::vector; using std::to_string; using std::ostream; @@ -14,7 +15,7 @@ const char *symbol_names[24] = { "twenty-two", "twenty-three" }; -SubtreeArray *tree_array(std::vector trees) { +SubtreeArray *tree_array(vector trees) { static SubtreeArray result; result.capacity = trees.size(); result.size = trees.size(); @@ -52,13 +53,13 @@ bool operator==(const TSNode &left, const TSNode &right) { ts_node_start_point(left) == ts_node_start_point(right); } -bool operator==(const std::vector &vec, const SubtreeArray &array) { +bool operator==(const vector &vec, const SubtreeArray &array) { return vec.size() == array.size && std::memcmp(vec.data(), array.contents, array.size * sizeof(Subtree)) == 0; } -void assert_consistent_tree_sizes(TSNode node) { +void assert_consistent_tree_sizes(TSNode node, const vector &line_starts) { uint32_t child_count = ts_node_child_count(node); uint32_t named_child_count = ts_node_named_child_count(node); uint32_t start_byte = ts_node_start_byte(node); @@ -69,6 +70,9 @@ void assert_consistent_tree_sizes(TSNode node) { AssertThat(start_byte, !IsGreaterThan(end_byte)); AssertThat(start_point, !IsGreaterThan(end_point)); + AssertThat(start_byte, Equals(line_starts[start_point.row] + start_point.column)); + AssertThat(end_byte, Equals(line_starts[end_point.row] + end_point.column)); + size_t last_child_end_byte = start_byte; TSPoint last_child_end_point = start_point; @@ -81,7 +85,7 @@ void assert_consistent_tree_sizes(TSNode node) { AssertThat(child_start_byte, !IsLessThan(last_child_end_byte)); AssertThat(child_start_point, !IsLessThan(last_child_end_point)); - assert_consistent_tree_sizes(child); + assert_consistent_tree_sizes(child, line_starts); if (ts_node_has_changes(child)) some_child_has_changes = true; if (ts_node_is_named(child)) actual_named_child_count++; @@ -101,3 +105,17 @@ void assert_consistent_tree_sizes(TSNode node) { AssertThat(ts_node_has_changes(node), IsTrue()); } } + +void assert_consistent_tree_sizes(const TSTree *tree, const string &text) { + vector line_starts; + line_starts.push_back(0); + for (uint32_t i = 0, n = text.size(); i < n; i++) { + if (text[i] == '\n') { + line_starts.push_back(i + 1); + } + } + + TSNode root_node = ts_tree_root_node(tree); + AssertThat(ts_node_end_byte(root_node), Equals(text.size())); + assert_consistent_tree_sizes(root_node, line_starts); +} diff --git a/test/helpers/tree_helpers.h b/test/helpers/tree_helpers.h index 1e836eef..2b59cea8 100644 --- a/test/helpers/tree_helpers.h +++ b/test/helpers/tree_helpers.h @@ -13,6 +13,6 @@ std::ostream &operator<<(std::ostream &stream, const TSNode &node); bool operator==(const TSNode &left, const TSNode &right); bool operator==(const std::vector &right, const SubtreeArray &array); -void assert_consistent_tree_sizes(TSNode node); +void assert_consistent_tree_sizes(const TSTree *, const std::string &); #endif // HELPERS_TREE_HELPERS_H_ diff --git a/test/integration/fuzzing-examples.cc b/test/integration/fuzzing-examples.cc index aca95840..555b4274 100644 --- a/test/integration/fuzzing-examples.cc +++ b/test/integration/fuzzing-examples.cc @@ -48,8 +48,7 @@ describe("examples found via fuzzing", [&]() { )); TSTree *tree = ts_parser_parse_string(parser, nullptr, input.c_str(), input.size()); - TSNode node = ts_tree_root_node(tree); - assert_consistent_tree_sizes(node); + assert_consistent_tree_sizes(tree, input); ts_tree_delete(tree); ts_parser_delete(parser); diff --git a/test/integration/real_grammars.cc b/test/integration/real_grammars.cc index 7384799c..6cacffde 100644 --- a/test/integration/real_grammars.cc +++ b/test/integration/real_grammars.cc @@ -11,12 +11,6 @@ #include "helpers/tree_helpers.h" #include -static void assert_correct_tree_size(TSTree *tree, string content) { - TSNode root_node = ts_tree_root_node(tree); - AssertThat(ts_node_end_byte(root_node), Equals(content.size())); - assert_consistent_tree_sizes(root_node); -} - START_TEST if (TREE_SITTER_SEED == -1) return; @@ -60,7 +54,7 @@ for (auto &language_name : test_languages) { if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); TSTree *tree = ts_parser_parse(parser, nullptr, input->input()); - assert_correct_tree_size(tree, input->content); + assert_consistent_tree_sizes(tree, input->content); TSNode root_node = ts_tree_root_node(tree); const char *node_string = ts_node_string(root_node); @@ -89,16 +83,16 @@ for (auto &language_name : test_languages) { input->replace(edit_position, 0, inserted_text); TSTree *tree = ts_parser_parse(parser, nullptr, input->input()); - assert_correct_tree_size(tree, input->content); + assert_consistent_tree_sizes(tree, input->content); if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); TSInputEdit edit = input->undo(); ts_tree_edit(tree, &edit); - assert_correct_tree_size(tree, input->content); + assert_consistent_tree_sizes(tree, input->content); if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); TSTree *new_tree = ts_parser_parse(parser, tree, input->input()); - assert_correct_tree_size(new_tree, input->content); + assert_consistent_tree_sizes(new_tree, input->content); uint32_t range_count; TSRange *ranges = ts_tree_get_changed_ranges(tree, new_tree, &range_count); @@ -132,16 +126,16 @@ for (auto &language_name : test_languages) { input->replace(edit_position, deletion_size, ""); TSTree *tree = ts_parser_parse(parser, nullptr, input->input()); - assert_correct_tree_size(tree, input->content); + assert_consistent_tree_sizes(tree, input->content); if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); TSInputEdit edit = input->undo(); ts_tree_edit(tree, &edit); - assert_correct_tree_size(tree, input->content); + assert_consistent_tree_sizes(tree, input->content); if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str()); TSTree *new_tree = ts_parser_parse(parser, tree, input->input()); - assert_correct_tree_size(new_tree, input->content); + assert_consistent_tree_sizes(new_tree, input->content); uint32_t range_count; TSRange *ranges = ts_tree_get_changed_ranges(tree, new_tree, &range_count); diff --git a/test/integration/test_grammars.cc b/test/integration/test_grammars.cc index d10523ae..7d3b6972 100644 --- a/test/integration/test_grammars.cc +++ b/test/integration/test_grammars.cc @@ -58,11 +58,9 @@ for (auto &language_name : test_languages) { } TSTree *tree = ts_parser_parse_string(parser, nullptr, entry.input.c_str(), entry.input.size()); + assert_consistent_tree_sizes(tree, entry.input); TSNode root_node = ts_tree_root_node(tree); - AssertThat(ts_node_end_byte(root_node), Equals(entry.input.size())); - assert_consistent_tree_sizes(root_node); - const char *node_string = ts_node_string(root_node); string result(node_string); ts_free((void *)node_string); diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc index 86abfea1..37e3b305 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/tree_test.cc @@ -56,10 +56,10 @@ describe("Tree", [&]() { input = new SpyInput(source_code, 32); TSTree *original_tree = ts_parser_parse(parser, nullptr, input->input()); - vector> new_trees; + vector>> new_trees; for (unsigned i = 0; i < 8; i++) { TSTree *tree_copy = ts_tree_copy(original_tree); - new_trees.push_back(std::async([i, tree_copy, &source_code, language]() { + new_trees.push_back(std::async([i, tree_copy, &source_code, language]() -> pair { Generator random(TREE_SITTER_SEED + i); TSTree *tree = tree_copy; @@ -83,9 +83,7 @@ describe("Tree", [&]() { } ts_parser_delete(parser); - delete input; - - return tree; + return {tree, input}; })); } @@ -93,9 +91,12 @@ describe("Tree", [&]() { for (auto &future : new_trees) { future.wait(); - TSTree *new_tree = future.get(); - assert_consistent_tree_sizes(ts_tree_root_node(new_tree)); + auto result = future.get(); + TSTree *new_tree = result.first; + SpyInput *new_input = result.second; + assert_consistent_tree_sizes(new_tree, new_input->content); ts_tree_delete(new_tree); + delete new_input; } }); });