traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add missing padding bytes check before storing tree inline

Browse source
This commit is contained in:
Max Brunsfeld 2018-09-19 10:52:21 -07:00
parent 3672da6ac3
commit 3dab0066bc
7 changed files with 41 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/runtime/subtree.c
View file

@ -160,6 +160,7 @@ static void ts_subtree_pool_free(SubtreePool *self, SubtreeHeapData *tree) {
static inline bool ts_subtree_can_inline(Length padding, Length size) {
return
padding.bytes < TS_MAX_INLINE_TREE_LENGTH &&
padding.extent.row < 16 &&
padding.extent.column < TS_MAX_INLINE_TREE_LENGTH &&
size.extent.row == 0 &&

26
test/helpers/tree_helpers.cc
View file

@ -4,6 +4,7 @@
#include <ostream>
using std::string;
using std::vector;
using std::to_string;
using std::ostream;
@ -14,7 +15,7 @@ const char *symbol_names[24] = {
"twenty-two", "twenty-three"
};
SubtreeArray *tree_array(std::vector<Subtree> trees) {
SubtreeArray *tree_array(vector<Subtree> trees) {
static SubtreeArray result;
result.capacity = trees.size();
result.size = trees.size();
@ -52,13 +53,13 @@ bool operator==(const TSNode &left, const TSNode &right) {
ts_node_start_point(left) == ts_node_start_point(right);
}
bool operator==(const std::vector<Subtree> &vec, const SubtreeArray &array) {
bool operator==(const vector<Subtree> &vec, const SubtreeArray &array) {
return
vec.size() == array.size &&
std::memcmp(vec.data(), array.contents, array.size * sizeof(Subtree)) == 0;
}
void assert_consistent_tree_sizes(TSNode node) {
void assert_consistent_tree_sizes(TSNode node, const vector<uint32_t> &line_starts) {
uint32_t child_count = ts_node_child_count(node);
uint32_t named_child_count = ts_node_named_child_count(node);
uint32_t start_byte = ts_node_start_byte(node);
@ -69,6 +70,9 @@ void assert_consistent_tree_sizes(TSNode node) {
AssertThat(start_byte, !IsGreaterThan(end_byte));
AssertThat(start_point, !IsGreaterThan(end_point));
AssertThat(start_byte, Equals(line_starts[start_point.row] + start_point.column));
AssertThat(end_byte, Equals(line_starts[end_point.row] + end_point.column));
size_t last_child_end_byte = start_byte;
TSPoint last_child_end_point = start_point;
@ -81,7 +85,7 @@ void assert_consistent_tree_sizes(TSNode node) {
AssertThat(child_start_byte, !IsLessThan(last_child_end_byte));
AssertThat(child_start_point, !IsLessThan(last_child_end_point));
assert_consistent_tree_sizes(child);
assert_consistent_tree_sizes(child, line_starts);
if (ts_node_has_changes(child)) some_child_has_changes = true;
if (ts_node_is_named(child)) actual_named_child_count++;
@ -101,3 +105,17 @@ void assert_consistent_tree_sizes(TSNode node) {
AssertThat(ts_node_has_changes(node), IsTrue());
}
}
void assert_consistent_tree_sizes(const TSTree *tree, const string &text) {
vector<uint32_t> line_starts;
line_starts.push_back(0);
for (uint32_t i = 0, n = text.size(); i < n; i++) {
if (text[i] == '\n') {
line_starts.push_back(i + 1);
}
}
TSNode root_node = ts_tree_root_node(tree);
AssertThat(ts_node_end_byte(root_node), Equals(text.size()));
assert_consistent_tree_sizes(root_node, line_starts);
}

2
test/helpers/tree_helpers.h
View file

@ -13,6 +13,6 @@ std::ostream &operator<<(std::ostream &stream, const TSNode &node);
bool operator==(const TSNode &left, const TSNode &right);
bool operator==(const std::vector<Subtree> &right, const SubtreeArray &array);
void assert_consistent_tree_sizes(TSNode node);
void assert_consistent_tree_sizes(const TSTree *, const std::string &);
#endif // HELPERS_TREE_HELPERS_H_

3
test/integration/fuzzing-examples.cc
View file

@ -48,8 +48,7 @@ describe("examples found via fuzzing", [&]() {
));
TSTree *tree = ts_parser_parse_string(parser, nullptr, input.c_str(), input.size());
TSNode node = ts_tree_root_node(tree);
assert_consistent_tree_sizes(node);
assert_consistent_tree_sizes(tree, input);
ts_tree_delete(tree);
ts_parser_delete(parser);

20
test/integration/real_grammars.cc
View file

@ -11,12 +11,6 @@
#include "helpers/tree_helpers.h"
#include <set>
static void assert_correct_tree_size(TSTree *tree, string content) {
TSNode root_node = ts_tree_root_node(tree);
AssertThat(ts_node_end_byte(root_node), Equals(content.size()));
assert_consistent_tree_sizes(root_node);
}
START_TEST
if (TREE_SITTER_SEED == -1) return;
@ -60,7 +54,7 @@ for (auto &language_name : test_languages) {
if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str());
TSTree *tree = ts_parser_parse(parser, nullptr, input->input());
assert_correct_tree_size(tree, input->content);
assert_consistent_tree_sizes(tree, input->content);
TSNode root_node = ts_tree_root_node(tree);
const char *node_string = ts_node_string(root_node);
@ -89,16 +83,16 @@ for (auto &language_name : test_languages) {
input->replace(edit_position, 0, inserted_text);
TSTree *tree = ts_parser_parse(parser, nullptr, input->input());
assert_correct_tree_size(tree, input->content);
assert_consistent_tree_sizes(tree, input->content);
if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str());
TSInputEdit edit = input->undo();
ts_tree_edit(tree, &edit);
assert_correct_tree_size(tree, input->content);
assert_consistent_tree_sizes(tree, input->content);
if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str());
TSTree *new_tree = ts_parser_parse(parser, tree, input->input());
assert_correct_tree_size(new_tree, input->content);
assert_consistent_tree_sizes(new_tree, input->content);
uint32_t range_count;
TSRange *ranges = ts_tree_get_changed_ranges(tree, new_tree, &range_count);
@ -132,16 +126,16 @@ for (auto &language_name : test_languages) {
input->replace(edit_position, deletion_size, "");
TSTree *tree = ts_parser_parse(parser, nullptr, input->input());
assert_correct_tree_size(tree, input->content);
assert_consistent_tree_sizes(tree, input->content);
if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str());
TSInputEdit edit = input->undo();
ts_tree_edit(tree, &edit);
assert_correct_tree_size(tree, input->content);
assert_consistent_tree_sizes(tree, input->content);
if (debug_graphs_enabled) printf("%s\n\n", input->content.c_str());
TSTree *new_tree = ts_parser_parse(parser, tree, input->input());
assert_correct_tree_size(new_tree, input->content);
assert_consistent_tree_sizes(new_tree, input->content);
uint32_t range_count;
TSRange *ranges = ts_tree_get_changed_ranges(tree, new_tree, &range_count);

4
test/integration/test_grammars.cc
View file

@ -58,11 +58,9 @@ for (auto &language_name : test_languages) {
}
TSTree *tree = ts_parser_parse_string(parser, nullptr, entry.input.c_str(), entry.input.size());
assert_consistent_tree_sizes(tree, entry.input);
TSNode root_node = ts_tree_root_node(tree);
AssertThat(ts_node_end_byte(root_node), Equals(entry.input.size()));
assert_consistent_tree_sizes(root_node);
const char *node_string = ts_node_string(root_node);
string result(node_string);
ts_free((void *)node_string);

15
test/runtime/tree_test.cc
View file

@ -56,10 +56,10 @@ describe("Tree", [&]() {
input = new SpyInput(source_code, 32);
TSTree *original_tree = ts_parser_parse(parser, nullptr, input->input());
vector<future<TSTree *>> new_trees;
vector<future<pair<TSTree *, SpyInput *>>> new_trees;
for (unsigned i = 0; i < 8; i++) {
TSTree *tree_copy = ts_tree_copy(original_tree);
new_trees.push_back(std::async([i, tree_copy, &source_code, language]() {
new_trees.push_back(std::async([i, tree_copy, &source_code, language]() -> pair<TSTree *, SpyInput *> {
Generator random(TREE_SITTER_SEED + i);
TSTree *tree = tree_copy;
@ -83,9 +83,7 @@ describe("Tree", [&]() {
}
ts_parser_delete(parser);
delete input;
return tree;
return {tree, input};
}));
}
@ -93,9 +91,12 @@ describe("Tree", [&]() {
for (auto &future : new_trees) {
future.wait();
TSTree *new_tree = future.get();
assert_consistent_tree_sizes(ts_tree_root_node(new_tree));
auto result = future.get();
TSTree *new_tree = result.first;
SpyInput *new_input = result.second;
assert_consistent_tree_sizes(new_tree, new_input->content);
ts_tree_delete(new_tree);
delete new_input;
}
});
});