traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Return a character count from the lexer's get_column method

Browse source
This commit is contained in:
Max Brunsfeld 2017-12-20 16:26:38 -08:00
parent fcff16cb86
commit 0e69da37a5
20 changed files with 143 additions and 233 deletions
Download patch file Download diff file Expand all files Collapse all files

18
test/runtime/document_test.cc
View file

@ -82,21 +82,12 @@ describe("Document", [&]() {
ts_document_parse(document);
});
it("allows columns to be measured in either bytes or characters", [&]() {
it("measures columns in bytes", [&]() {
const char16_t content[] = u"[true, false]";
spy_input->content = string((const char *)content, sizeof(content));
spy_input->encoding = TSInputEncodingUTF16;
TSInput input = spy_input->input();
input.measure_columns_in_bytes = false;
ts_document_set_input(document, input);
ts_document_invalidate(document);
ts_document_parse(document);
TSNode root = ts_document_root_node(document);
AssertThat(ts_node_end_point(root), Equals<TSPoint>({0, 14}));
input.measure_columns_in_bytes = true;
ts_document_set_input(document, input);
ts_document_invalidate(document);
ts_document_parse(document);
@ -142,7 +133,7 @@ describe("Document", [&]() {
ts_document_set_input_string_with_length(document, content, 1);
ts_document_parse(document);
TSNode new_root = ts_document_root_node(document);
AssertThat(ts_node_end_char(new_root), Equals<size_t>(1));
AssertThat(ts_node_end_byte(new_root), Equals<size_t>(1));
assert_node_string_equals(
new_root,
"(value (number))");
@ -152,7 +143,7 @@ describe("Document", [&]() {
ts_document_set_input_string(document, "");
ts_document_parse(document);
TSNode new_root = ts_document_root_node(document);
AssertThat(ts_node_end_char(new_root), Equals<size_t>(0));
AssertThat(ts_node_end_byte(new_root), Equals<size_t>(0));
assert_node_string_equals(
new_root,
"(ERROR)");
@ -160,7 +151,7 @@ describe("Document", [&]() {
ts_document_set_input_string(document, "1");
ts_document_parse(document);
new_root = ts_document_root_node(document);
AssertThat(ts_node_end_char(new_root), Equals<size_t>(1));
AssertThat(ts_node_end_byte(new_root), Equals<size_t>(1));
assert_node_string_equals(
new_root,
"(value (number))");
@ -445,7 +436,6 @@ describe("Document", [&]() {
root,
"(ERROR (number) (null) (UNEXPECTED 'e'))");
AssertThat(ts_node_end_char(root), Equals(input_string.size()));
AssertThat(ts_node_end_byte(root), Equals(input_string.size()));
});

40
test/runtime/node_test.cc
View file

@ -99,8 +99,6 @@ describe("Node", [&]() {
AssertThat(ts_node_named_child_count(root_node), Equals<size_t>(3));
AssertThat(ts_node_start_byte(root_node), Equals(array_index));
AssertThat(ts_node_end_byte(root_node), Equals(array_end_index));
AssertThat(ts_node_start_char(root_node), Equals(array_index));
AssertThat(ts_node_end_char(root_node), Equals(array_end_index));
AssertThat(ts_node_start_point(root_node), Equals<TSPoint>({ 2, 0 }));
AssertThat(ts_node_end_point(root_node), Equals<TSPoint>({ 8, 1 }));
@ -114,8 +112,6 @@ describe("Node", [&]() {
AssertThat(ts_node_start_byte(number_node), Equals(number_index));
AssertThat(ts_node_end_byte(number_node), Equals(number_end_index));
AssertThat(ts_node_start_char(number_node), Equals(number_index));
AssertThat(ts_node_end_char(number_node), Equals(number_end_index));
AssertThat(ts_node_start_point(number_node), Equals<TSPoint>({ 3, 2 }));
AssertThat(ts_node_end_point(number_node), Equals<TSPoint>({ 3, 5 }));
@ -191,7 +187,7 @@ describe("Node", [&]() {
it("returns an iterator that yields each of the node's symbols", [&]() {
const TSLanguage *language = ts_document_language(document);
TSNode false_node = ts_node_descendant_for_char_range(root_node, false_index, false_index + 1);
TSNode false_node = ts_node_descendant_for_byte_range(root_node, false_index, false_index + 1);
TSSymbolIterator iterator = ts_node_symbols(false_node);
AssertThat(iterator.done, Equals(false));
AssertThat(ts_language_symbol_name(language, iterator.value), Equals("false"));
@ -203,7 +199,7 @@ describe("Node", [&]() {
ts_symbol_iterator_next(&iterator);
AssertThat(iterator.done, Equals(true));
TSNode comma_node = ts_node_descendant_for_char_range(root_node, number_end_index, number_end_index);
TSNode comma_node = ts_node_descendant_for_byte_range(root_node, number_end_index, number_end_index);
iterator = ts_node_symbols(comma_node);
AssertThat(iterator.done, Equals(false));
AssertThat(ts_language_symbol_name(language, iterator.value), Equals(","));
@ -375,17 +371,17 @@ describe("Node", [&]() {
});
});
describe("named_descendant_for_char_range(start, end)", [&]() {
describe("named_descendant_for_byte_range(start, end)", [&]() {
describe("when there is a leaf node that spans the given range exactly", [&]() {
it("returns that leaf node", [&]() {
TSNode leaf = ts_node_named_descendant_for_char_range(root_node, string_index, string_end_index - 1);
TSNode leaf = ts_node_named_descendant_for_byte_range(root_node, string_index, string_end_index - 1);
AssertThat(ts_node_type(leaf, document), Equals("string"));
AssertThat(ts_node_start_byte(leaf), Equals(string_index));
AssertThat(ts_node_end_byte(leaf), Equals(string_end_index));
AssertThat(ts_node_start_point(leaf), Equals<TSPoint>({ 6, 4 }));
AssertThat(ts_node_end_point(leaf), Equals<TSPoint>({ 6, 7 }));
leaf = ts_node_named_descendant_for_char_range(root_node, number_index, number_end_index - 1);
leaf = ts_node_named_descendant_for_byte_range(root_node, number_index, number_end_index - 1);
AssertThat(ts_node_type(leaf, document), Equals("number"));
AssertThat(ts_node_start_byte(leaf), Equals(number_index));
AssertThat(ts_node_end_byte(leaf), Equals(number_end_index));
@ -396,14 +392,14 @@ describe("Node", [&]() {
describe("when there is a leaf node that extends beyond the given range", [&]() {
it("returns that leaf node", [&]() {
TSNode leaf = ts_node_named_descendant_for_char_range(root_node, string_index, string_index + 1);
TSNode leaf = ts_node_named_descendant_for_byte_range(root_node, string_index, string_index + 1);
AssertThat(ts_node_type(leaf, document), Equals("string"));
AssertThat(ts_node_start_byte(leaf), Equals(string_index));
AssertThat(ts_node_end_byte(leaf), Equals(string_end_index));
AssertThat(ts_node_start_point(leaf), Equals<TSPoint>({ 6, 4 }));
AssertThat(ts_node_end_point(leaf), Equals<TSPoint>({ 6, 7 }));
leaf = ts_node_named_descendant_for_char_range(root_node, string_index + 1, string_index + 2);
leaf = ts_node_named_descendant_for_byte_range(root_node, string_index + 1, string_index + 2);
AssertThat(ts_node_type(leaf, document), Equals("string"));
AssertThat(ts_node_start_byte(leaf), Equals(string_index));
AssertThat(ts_node_end_byte(leaf), Equals(string_end_index));
@ -414,7 +410,7 @@ describe("Node", [&]() {
describe("when there is no leaf node that spans the given range", [&]() {
it("returns the smallest node that does span the range", [&]() {
TSNode pair_node = ts_node_named_descendant_for_char_range(root_node, string_index, string_index + 3);
TSNode pair_node = ts_node_named_descendant_for_byte_range(root_node, string_index, string_index + 3);
AssertThat(ts_node_type(pair_node, document), Equals("pair"));
AssertThat(ts_node_start_byte(pair_node), Equals(string_index));
AssertThat(ts_node_end_byte(pair_node), Equals(null_end_index));
@ -423,7 +419,7 @@ describe("Node", [&]() {
});
it("does not return invisible nodes (repeats)", [&]() {
TSNode node = ts_node_named_descendant_for_char_range(root_node, number_end_index, number_end_index + 1);
TSNode node = ts_node_named_descendant_for_byte_range(root_node, number_end_index, number_end_index + 1);
AssertThat(ts_node_type(node, document), Equals("array"));
AssertThat(ts_node_start_byte(node), Equals(array_index));
AssertThat(ts_node_end_byte(node), Equals(array_end_index));
@ -433,31 +429,31 @@ describe("Node", [&]() {
});
});
describe("descendant_for_char_range(start, end)", [&]() {
it("returns the smallest node that spans the given range", [&]() {
TSNode node1 = ts_node_descendant_for_char_range(root_node, colon_index, colon_index);
describe("descendant_for_byte_range(start, end)", [&]() {
it("returns the smallest node that spans the given byte offsets", [&]() {
TSNode node1 = ts_node_descendant_for_byte_range(root_node, colon_index, colon_index);
AssertThat(ts_node_type(node1, document), Equals(":"));
AssertThat(ts_node_start_byte(node1), Equals(colon_index));
AssertThat(ts_node_end_byte(node1), Equals(colon_index + 1));
AssertThat(ts_node_start_point(node1), Equals<TSPoint>({ 6, 7 }));
AssertThat(ts_node_end_point(node1), Equals<TSPoint>({ 6, 8 }));
TSNode node2 = ts_node_descendant_for_char_range(root_node, string_index + 2, string_index + 4);
TSNode node2 = ts_node_descendant_for_byte_range(root_node, string_index + 2, string_index + 4);
AssertThat(ts_node_type(node2, document), Equals("pair"));
AssertThat(ts_node_start_byte(node2), Equals(string_index));
AssertThat(ts_node_end_byte(node2), Equals(null_end_index));
AssertThat(ts_node_start_point(node2), Equals<TSPoint>({ 6, 4 }));
AssertThat(ts_node_end_point(node2), Equals<TSPoint>({ 6, 13 }));
});
});
describe("descendant_for_byte_range(start, end)", [&]() {
it("returns the smallest concrete node that spans the given range", [&]() {
ts_document_set_input_string(document, "[\"αβγδ\", \"αβγδ\"]");
it("works in the presence of multi-byte characters", [&]() {
string input_string = "[\"αβγδ\", \"αβγδ\"]";
ts_document_set_input_string(document, input_string.c_str());
ts_document_parse(document);
TSNode root_node = ts_document_root_node(document);
TSNode node1 = ts_node_descendant_for_char_range(root_node, 7, 7);
uint32_t comma_position = input_string.find(",");
TSNode node1 = ts_node_descendant_for_byte_range(root_node, comma_position, comma_position);
AssertThat(ts_node_type(node1, document), Equals(","));
TSNode node2 = ts_node_descendant_for_byte_range(root_node, 6, 10);

5
test/runtime/parser_test.cc
View file

@ -335,7 +335,7 @@ describe("Parser", [&]() {
assert_root_node(
"(program (expression_statement (binary_expression (identifier) (number))))");
TSNode node = ts_node_named_descendant_for_char_range(root, 1, 1);
TSNode node = ts_node_named_descendant_for_byte_range(root, 1, 1);
AssertThat(ts_node_type(node, document), Equals("identifier"));
AssertThat(ts_node_end_byte(node), Equals(strlen("abXYZc")));
});
@ -354,7 +354,7 @@ describe("Parser", [&]() {
assert_root_node(
"(program (expression_statement (binary_expression (identifier) (number))))");
TSNode node = ts_node_named_descendant_for_char_range(root, 1, 1);
TSNode node = ts_node_named_descendant_for_byte_range(root, 1, 1);
AssertThat(ts_node_type(node, document), Equals("identifier"));
AssertThat(ts_node_end_byte(node), Equals(strlen("abcXYZ")));
});
@ -487,7 +487,6 @@ describe("Parser", [&]() {
assert_root_node(
"(program (expression_statement (string)))");
AssertThat(ts_node_end_char(root), Equals(strlen("'OOO - DD';")));
AssertThat(ts_node_end_byte(root), Equals(strlen("'\u03A9\u03A9\u03A9 \u2014 \u0394\u0394';")));
});

4
test/runtime/stack_test.cc
View file

@ -20,7 +20,7 @@ enum {
};
Length operator*(const Length &length, uint32_t factor) {
return {length.bytes * factor, length.chars * factor, {0, length.extent.column * factor}};
return {length.bytes * factor, {0, length.extent.column * factor}};
}
void free_slice_array(StackSliceArray *slices) {
@ -71,7 +71,7 @@ describe("Stack", [&]() {
Stack *stack;
const size_t tree_count = 11;
Tree *trees[tree_count];
Length tree_len = {2, 3, {0, 3}};
Length tree_len = {3, {0, 3}};
before_each([&]() {
record_alloc::start();

85
test/runtime/tree_test.cc
View file

@ -42,7 +42,7 @@ describe("Tree", []() {
describe("make_leaf", [&]() {
it("does not mark the tree as fragile", [&]() {
Tree *tree = ts_tree_make_leaf(symbol1, {2, 1, {0, 1}}, {5, 4, {0, 4}}, &language);
Tree *tree = ts_tree_make_leaf(symbol1, {2, {0, 1}}, {5, {0, 4}}, &language);
AssertThat(tree->fragile_left, IsFalse());
AssertThat(tree->fragile_right, IsFalse());
});
@ -68,8 +68,8 @@ describe("Tree", []() {
Tree *tree1, *tree2, *parent1;
before_each([&]() {
tree1 = ts_tree_make_leaf(symbol1, {2, 1, {0, 1}}, {5, 4, {0, 4}}, &language);
tree2 = ts_tree_make_leaf(symbol2, {1, 1, {0, 1}}, {3, 3, {0, 3}}, &language);
tree1 = ts_tree_make_leaf(symbol1, {2, {0, 1}}, {5, {0, 4}}, &language);
tree2 = ts_tree_make_leaf(symbol2, {1, {0, 1}}, {3, {0, 3}}, &language);
ts_tree_retain(tree1);
ts_tree_retain(tree2);
@ -87,12 +87,9 @@ describe("Tree", []() {
it("computes its size and padding based on its child nodes", [&]() {
AssertThat(parent1->size.bytes, Equals<size_t>(
tree1->size.bytes + tree2->padding.bytes + tree2->size.bytes));
AssertThat(parent1->size.chars, Equals<size_t>(
tree1->size.chars + tree2->padding.chars + tree2->size.chars));
tree1->size.bytes + tree2->padding.bytes + tree2->size.bytes
));
AssertThat(parent1->padding.bytes, Equals<size_t>(tree1->padding.bytes));
AssertThat(parent1->padding.chars, Equals<size_t>(tree1->padding.chars));
});
describe("when the first node is fragile on the left side", [&]() {
@ -174,13 +171,13 @@ describe("Tree", []() {
before_each([&]() {
tree = ts_tree_make_node(symbol1, 3, tree_array({
ts_tree_make_leaf(symbol2, {2, 2, {0, 2}}, {3, 3, {0, 3}}, &language),
ts_tree_make_leaf(symbol3, {2, 2, {0, 2}}, {3, 3, {0, 3}}, &language),
ts_tree_make_leaf(symbol4, {2, 2, {0, 2}}, {3, 3, {0, 3}}, &language),
ts_tree_make_leaf(symbol2, {2, {0, 2}}, {3, {0, 3}}, &language),
ts_tree_make_leaf(symbol3, {2, {0, 2}}, {3, {0, 3}}, &language),
ts_tree_make_leaf(symbol4, {2, {0, 2}}, {3, {0, 3}}, &language),
}), 0, &language);
AssertThat(tree->padding, Equals<Length>({2, 2, {0, 2}}));
AssertThat(tree->size, Equals<Length>({13, 13, {0, 13}}));
AssertThat(tree->padding, Equals<Length>({2, {0, 2}}));
AssertThat(tree->size, Equals<Length>({13, {0, 13}}));
});
after_each([&]() {
@ -200,16 +197,16 @@ describe("Tree", []() {
assert_consistent(tree);
AssertThat(tree->has_changes, IsTrue());
AssertThat(tree->padding, Equals<Length>({3, 0, {0, 3}}));
AssertThat(tree->size, Equals<Length>({13, 13, {0, 13}}));
AssertThat(tree->padding, Equals<Length>({3, {0, 3}}));
AssertThat(tree->size, Equals<Length>({13, {0, 13}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<Length>({3, 0, {0, 3}}));
AssertThat(tree->children[0]->size, Equals<Length>({3, 3, {0, 3}}));
AssertThat(tree->children[0]->padding, Equals<Length>({3, {0, 3}}));
AssertThat(tree->children[0]->size, Equals<Length>({3, {0, 3}}));
AssertThat(tree->children[1]->has_changes, IsFalse());
AssertThat(tree->children[1]->padding, Equals<Length>({2, 2, {0, 2}}));
AssertThat(tree->children[1]->size, Equals<Length>({3, 3, {0, 3}}));
AssertThat(tree->children[1]->padding, Equals<Length>({2, {0, 2}}));
AssertThat(tree->children[1]->size, Equals<Length>({3, {0, 3}}));
});
});
@ -226,12 +223,12 @@ describe("Tree", []() {
assert_consistent(tree);
AssertThat(tree->has_changes, IsTrue());
AssertThat(tree->padding, Equals<Length>({5, 0, {0, 5}}));
AssertThat(tree->size, Equals<Length>({11, 0, {0, 11}}));
AssertThat(tree->padding, Equals<Length>({5, {0, 5}}));
AssertThat(tree->size, Equals<Length>({11, {0, 11}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<Length>({5, 0, {0, 5}}));
AssertThat(tree->children[0]->size, Equals<Length>({1, 0, {0, 1}}));
AssertThat(tree->children[0]->padding, Equals<Length>({5, {0, 5}}));
AssertThat(tree->children[0]->size, Equals<Length>({1, {0, 1}}));
});
});
@ -250,12 +247,12 @@ describe("Tree", []() {
assert_consistent(tree);
AssertThat(tree->has_changes, IsTrue());
AssertThat(tree->padding, Equals<Length>({4, 0, {0, 4}}));
AssertThat(tree->size, Equals<Length>({13, 13, {0, 13}}));
AssertThat(tree->padding, Equals<Length>({4, {0, 4}}));
AssertThat(tree->size, Equals<Length>({13, {0, 13}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<Length>({4, 0, {0, 4}}));
AssertThat(tree->children[0]->size, Equals<Length>({3, 3, {0, 3}}));
AssertThat(tree->children[0]->padding, Equals<Length>({4, {0, 4}}));
AssertThat(tree->children[0]->size, Equals<Length>({3, {0, 3}}));
AssertThat(tree->children[1]->has_changes, IsFalse());
});
@ -274,12 +271,12 @@ describe("Tree", []() {
assert_consistent(tree);
AssertThat(tree->has_changes, IsTrue());
AssertThat(tree->padding, Equals<Length>({2, 2, {0, 2}}));
AssertThat(tree->size, Equals<Length>({16, 0, {0, 16}}));
AssertThat(tree->padding, Equals<Length>({2, {0, 2}}));
AssertThat(tree->size, Equals<Length>({16, {0, 16}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<Length>({2, 2, {0, 2}}));
AssertThat(tree->children[0]->size, Equals<Length>({6, 0, {0, 6}}));
AssertThat(tree->children[0]->padding, Equals<Length>({2, {0, 2}}));
AssertThat(tree->children[0]->size, Equals<Length>({6, {0, 6}}));
AssertThat(tree->children[1]->has_changes, IsFalse());
});
@ -300,20 +297,20 @@ describe("Tree", []() {
assert_consistent(tree);
AssertThat(tree->has_changes, IsTrue());
AssertThat(tree->padding, Equals<Length>({4, 0, {0, 4}}));
AssertThat(tree->size, Equals<Length>({4, 0, {0, 4}}));
AssertThat(tree->padding, Equals<Length>({4, {0, 4}}));
AssertThat(tree->size, Equals<Length>({4, {0, 4}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<Length>({4, 0, {0, 4}}));
AssertThat(tree->children[0]->size, Equals<Length>({0, 0, {0, 0}}));
AssertThat(tree->children[0]->padding, Equals<Length>({4, {0, 4}}));
AssertThat(tree->children[0]->size, Equals<Length>({0, {0, 0}}));
AssertThat(tree->children[1]->has_changes, IsTrue());
AssertThat(tree->children[1]->padding, Equals<Length>({0, 0, {0, 0}}));
AssertThat(tree->children[1]->size, Equals<Length>({0, 0, {0, 0}}));
AssertThat(tree->children[1]->padding, Equals<Length>({0, {0, 0}}));
AssertThat(tree->children[1]->size, Equals<Length>({0, {0, 0}}));
AssertThat(tree->children[2]->has_changes, IsTrue());
AssertThat(tree->children[2]->padding, Equals<Length>({1, 0, {0, 1}}));
AssertThat(tree->children[2]->size, Equals<Length>({3, 3, {0, 3}}));
AssertThat(tree->children[2]->padding, Equals<Length>({1, {0, 1}}));
AssertThat(tree->children[2]->size, Equals<Length>({3, {0, 3}}));
});
});
@ -340,7 +337,7 @@ describe("Tree", []() {
Tree *leaf;
before_each([&]() {
leaf = ts_tree_make_leaf(symbol1, {2, 1, {0, 1}}, {5, 4, {0, 4}}, &language);
leaf = ts_tree_make_leaf(symbol1, {2, {0, 1}}, {5, {0, 4}}, &language);
});
after_each([&]() {
@ -348,7 +345,7 @@ describe("Tree", []() {
});
it("returns true for identical trees", [&]() {
Tree *leaf_copy = ts_tree_make_leaf(symbol1, {2, 1, {1, 1}}, {5, 4, {1, 4}}, &language);
Tree *leaf_copy = ts_tree_make_leaf(symbol1, {2, {1, 1}}, {5, {1, 4}}, &language);
AssertThat(ts_tree_eq(leaf, leaf_copy), IsTrue());
Tree *parent = ts_tree_make_node(symbol2, 2, tree_array({
@ -402,7 +399,7 @@ describe("Tree", []() {
});
it("returns false for trees with different children", [&]() {
Tree *leaf2 = ts_tree_make_leaf(symbol2, {1, 1, {0, 1}}, {3, 3, {0, 3}}, &language);
Tree *leaf2 = ts_tree_make_leaf(symbol2, {1, {0, 1}}, {3, {0, 3}}, &language);
Tree *parent = ts_tree_make_node(symbol2, 2, tree_array({
leaf,
@ -428,8 +425,8 @@ describe("Tree", []() {
});
describe("last_external_token", [&]() {
Length padding = {1, 1, {0, 1}};
Length size = {2, 2, {0, 2}};
Length padding = {1, {0, 1}};
Length size = {2, {0, 2}};
auto make_external = [](Tree *tree) {
tree->has_external_tokens = true;