Fix incorrect node renames in the presence of extra tokens

This commit is contained in:
Max Brunsfeld 2017-07-18 21:24:34 -07:00
parent 10d28d4b56
commit f33421c53e
3 changed files with 164 additions and 102 deletions

View file

@ -8,50 +8,70 @@
START_TEST
describe("Node", []() {
TSDocument *document;
TSNode array_node;
string input_string =
"\n"
"\n"
"[\n"
" 123,\n"
" false,\n"
" {\n"
" \"x\": null\n"
" }\n"
"]";
string json_string = R"JSON(
size_t array_index = input_string.find("[\n");
size_t array_end_index = input_string.find("]") + 1;
size_t number_index = input_string.find("123");
size_t number_end_index = number_index + string("123").size();
size_t false_index = input_string.find("false");
size_t false_end_index = false_index + string("false").size();
size_t object_index = input_string.find("{");
size_t object_end_index = input_string.find("}") + 1;
size_t string_index = input_string.find("\"x\"");
size_t string_end_index = string_index + 3;
size_t colon_index = input_string.find(":");
size_t null_index = input_string.find("null");
size_t null_end_index = null_index + string("null").size();
[
123,
false,
{
"x": null
}
]
)JSON";
size_t array_index = json_string.find("[\n");
size_t array_end_index = json_string.find("]") + 1;
size_t number_index = json_string.find("123");
size_t number_end_index = number_index + string("123").size();
size_t false_index = json_string.find("false");
size_t false_end_index = false_index + string("false").size();
size_t object_index = json_string.find("{");
size_t object_end_index = json_string.find("}") + 1;
size_t string_index = json_string.find("\"x\"");
size_t string_end_index = string_index + 3;
size_t colon_index = json_string.find(":");
size_t null_index = json_string.find("null");
size_t null_end_index = null_index + string("null").size();
string grammar_with_renames_and_extras = R"JSON({
"name": "renames_and_extras",
"extras": [
{"type": "PATTERN", "value": "\\s+"},
{"type": "SYMBOL", "name": "comment"},
],
"rules": {
"a": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "b"},
{
"type": "RENAME",
"value": "B",
"content": {"type": "SYMBOL", "name": "b"}
},
{"type": "SYMBOL", "name": "b"}
]
},
"b": {"type": "STRING", "value": "b"},
"comment": {"type": "STRING", "value": "..."}
}
})JSON";
describe("Node", [&]() {
TSDocument *document;
TSNode root_node;
before_each([&]() {
record_alloc::start();
document = ts_document_new();
ts_document_set_language(document, load_real_language("json"));
ts_document_set_input_string(document, input_string.c_str());
ts_document_set_input_string(document, json_string.c_str());
ts_document_parse(document);
array_node = ts_document_root_node(document);
char *node_string = ts_node_string(array_node, document);
AssertThat(node_string, Equals(
"(array "
"(number) "
"(false) "
"(object (pair (string) (null))))"));
ts_free(node_string);
root_node = ts_document_root_node(document);
});
after_each([&]() {
@ -61,21 +81,31 @@ describe("Node", []() {
AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty());
});
it("parses the example as expected (precondition)", [&]() {
char *node_string = ts_node_string(root_node, document);
AssertThat(node_string, Equals(
"(array "
"(number) "
"(false) "
"(object (pair (string) (null))))"));
ts_free(node_string);
});
describe("named_child_count(), named_child(i)", [&]() {
it("returns the named child node at the given index", [&]() {
AssertThat(ts_node_type(array_node, document), Equals("array"));
AssertThat(ts_node_type(root_node, document), Equals("array"));
AssertThat(ts_node_named_child_count(array_node), Equals<size_t>(3));
AssertThat(ts_node_start_byte(array_node), Equals(array_index));
AssertThat(ts_node_end_byte(array_node), Equals(array_end_index));
AssertThat(ts_node_start_char(array_node), Equals(array_index));
AssertThat(ts_node_end_char(array_node), Equals(array_end_index));
AssertThat(ts_node_start_point(array_node), Equals<TSPoint>({ 2, 0 }));
AssertThat(ts_node_end_point(array_node), Equals<TSPoint>({ 8, 1 }));
AssertThat(ts_node_named_child_count(root_node), Equals<size_t>(3));
AssertThat(ts_node_start_byte(root_node), Equals(array_index));
AssertThat(ts_node_end_byte(root_node), Equals(array_end_index));
AssertThat(ts_node_start_char(root_node), Equals(array_index));
AssertThat(ts_node_end_char(root_node), Equals(array_end_index));
AssertThat(ts_node_start_point(root_node), Equals<TSPoint>({ 2, 0 }));
AssertThat(ts_node_end_point(root_node), Equals<TSPoint>({ 8, 1 }));
TSNode number_node = ts_node_named_child(array_node, 0);
TSNode false_node = ts_node_named_child(array_node, 1);
TSNode object_node = ts_node_named_child(array_node, 2);
TSNode number_node = ts_node_named_child(root_node, 0);
TSNode false_node = ts_node_named_child(root_node, 1);
TSNode object_node = ts_node_named_child(root_node, 2);
AssertThat(ts_node_type(number_node, document), Equals("number"));
AssertThat(ts_node_type(false_node, document), Equals("false"));
@ -127,10 +157,32 @@ describe("Node", []() {
AssertThat(ts_node_parent(string_node), Equals(pair_node));
AssertThat(ts_node_parent(null_node), Equals(pair_node));
AssertThat(ts_node_parent(pair_node), Equals(object_node));
AssertThat(ts_node_parent(number_node), Equals(array_node));
AssertThat(ts_node_parent(false_node), Equals(array_node));
AssertThat(ts_node_parent(object_node), Equals(array_node));
AssertThat(ts_node_parent(array_node).data, Equals<void *>(nullptr));
AssertThat(ts_node_parent(number_node), Equals(root_node));
AssertThat(ts_node_parent(false_node), Equals(root_node));
AssertThat(ts_node_parent(object_node), Equals(root_node));
AssertThat(ts_node_parent(root_node).data, Equals<void *>(nullptr));
});
it("works correctly when the node contains renamed children and extras", [&]() {
TSCompileResult compile_result = ts_compile_grammar(grammar_with_renames_and_extras.c_str());
const TSLanguage *language = load_test_language("renames_and_extras", compile_result);
ts_document_set_language(document, language);
ts_document_set_input_string(document, "b ... b ... b");
ts_document_parse(document);
root_node = ts_document_root_node(document);
char *node_string = ts_node_string(root_node, document);
AssertThat(node_string, Equals("(a (b) (comment) (B) (comment) (b))"));
ts_free(node_string);
AssertThat(ts_node_named_child_count(root_node), Equals(5u));
AssertThat(ts_node_type(ts_node_named_child(root_node, 0), document), Equals("b"));
AssertThat(ts_node_type(ts_node_named_child(root_node, 1), document), Equals("comment"));
AssertThat(ts_node_type(ts_node_named_child(root_node, 2), document), Equals("B"));
AssertThat(ts_node_type(ts_node_named_child(root_node, 3), document), Equals("comment"));
AssertThat(ts_node_type(ts_node_named_child(root_node, 4), document), Equals("b"));
AssertThat(ts_node_symbol(ts_node_named_child(root_node, 0)), !Equals(ts_node_symbol(ts_node_named_child(root_node, 2))));
});
});
@ -138,7 +190,7 @@ describe("Node", []() {
it("returns an iterator that yields each of the node's symbols", [&]() {
const TSLanguage *language = ts_document_language(document);
TSNode false_node = ts_node_descendant_for_char_range(array_node, false_index, false_index + 1);
TSNode false_node = ts_node_descendant_for_char_range(root_node, false_index, false_index + 1);
TSSymbolIterator iterator = ts_node_symbols(false_node);
AssertThat(iterator.done, Equals(false));
AssertThat(ts_language_symbol_name(language, iterator.value), Equals("false"));
@ -150,7 +202,7 @@ describe("Node", []() {
ts_symbol_iterator_next(&iterator);
AssertThat(iterator.done, Equals(true));
TSNode comma_node = ts_node_descendant_for_char_range(array_node, number_end_index, number_end_index);
TSNode comma_node = ts_node_descendant_for_char_range(root_node, number_end_index, number_end_index);
iterator = ts_node_symbols(comma_node);
AssertThat(iterator.done, Equals(false));
AssertThat(ts_language_symbol_name(language, iterator.value), Equals(","));
@ -162,16 +214,16 @@ describe("Node", []() {
describe("child_count(), child(i)", [&]() {
it("returns the child node at the given index, including anonymous nodes", [&]() {
AssertThat(ts_node_child_count(array_node), Equals<size_t>(7));
TSNode child1 = ts_node_child(array_node, 0);
TSNode child2 = ts_node_child(array_node, 1);
TSNode child3 = ts_node_child(array_node, 2);
TSNode child4 = ts_node_child(array_node, 3);
TSNode child5 = ts_node_child(array_node, 4);
TSNode child6 = ts_node_child(array_node, 5);
TSNode child7 = ts_node_child(array_node, 6);
AssertThat(ts_node_child_count(root_node), Equals<size_t>(7));
TSNode child1 = ts_node_child(root_node, 0);
TSNode child2 = ts_node_child(root_node, 1);
TSNode child3 = ts_node_child(root_node, 2);
TSNode child4 = ts_node_child(root_node, 3);
TSNode child5 = ts_node_child(root_node, 4);
TSNode child6 = ts_node_child(root_node, 5);
TSNode child7 = ts_node_child(root_node, 6);
AssertThat(ts_node_type(array_node, document), Equals("array"));
AssertThat(ts_node_type(root_node, document), Equals("array"));
AssertThat(ts_node_type(child1, document), Equals("["));
AssertThat(ts_node_type(child2, document), Equals("number"));
AssertThat(ts_node_type(child3, document), Equals(","));
@ -180,7 +232,7 @@ describe("Node", []() {
AssertThat(ts_node_type(child6, document), Equals("object"));
AssertThat(ts_node_type(child7, document), Equals("]"));
AssertThat(ts_node_is_named(array_node), IsTrue());
AssertThat(ts_node_is_named(root_node), IsTrue());
AssertThat(ts_node_is_named(child1), IsFalse());
AssertThat(ts_node_is_named(child2), IsTrue());
AssertThat(ts_node_is_named(child3), IsFalse());
@ -233,32 +285,32 @@ describe("Node", []() {
AssertThat(ts_node_parent(left_brace), Equals(child6));
AssertThat(ts_node_parent(pair), Equals(child6));
AssertThat(ts_node_parent(right_brace), Equals(child6));
AssertThat(ts_node_parent(child1), Equals(array_node));
AssertThat(ts_node_parent(child2), Equals(array_node));
AssertThat(ts_node_parent(child3), Equals(array_node));
AssertThat(ts_node_parent(child4), Equals(array_node));
AssertThat(ts_node_parent(child5), Equals(array_node));
AssertThat(ts_node_parent(child6), Equals(array_node));
AssertThat(ts_node_parent(child7), Equals(array_node));
AssertThat(ts_node_parent(array_node).data, Equals<void *>(nullptr));
AssertThat(ts_node_parent(child1), Equals(root_node));
AssertThat(ts_node_parent(child2), Equals(root_node));
AssertThat(ts_node_parent(child3), Equals(root_node));
AssertThat(ts_node_parent(child4), Equals(root_node));
AssertThat(ts_node_parent(child5), Equals(root_node));
AssertThat(ts_node_parent(child6), Equals(root_node));
AssertThat(ts_node_parent(child7), Equals(root_node));
AssertThat(ts_node_parent(root_node).data, Equals<void *>(nullptr));
});
});
describe("next_sibling(), prev_sibling()", [&]() {
it("returns the node's next and previous sibling, including anonymous nodes", [&]() {
TSNode bracket_node1 = ts_node_child(array_node, 0);
TSNode number_node = ts_node_child(array_node, 1);
TSNode array_comma_node1 = ts_node_child(array_node, 2);
TSNode false_node = ts_node_child(array_node, 3);
TSNode array_comma_node2 = ts_node_child(array_node, 4);
TSNode object_node = ts_node_child(array_node, 5);
TSNode bracket_node1 = ts_node_child(root_node, 0);
TSNode number_node = ts_node_child(root_node, 1);
TSNode array_comma_node1 = ts_node_child(root_node, 2);
TSNode false_node = ts_node_child(root_node, 3);
TSNode array_comma_node2 = ts_node_child(root_node, 4);
TSNode object_node = ts_node_child(root_node, 5);
TSNode brace_node1 = ts_node_child(object_node, 0);
TSNode pair_node = ts_node_child(object_node, 1);
TSNode string_node = ts_node_child(pair_node, 0);
TSNode colon_node = ts_node_child(pair_node, 1);
TSNode null_node = ts_node_child(pair_node, 2);
TSNode brace_node2 = ts_node_child(object_node, 2);
TSNode bracket_node2 = ts_node_child(array_node, 6);
TSNode bracket_node2 = ts_node_child(root_node, 6);
AssertThat(ts_node_next_sibling(bracket_node1), Equals(number_node));
AssertThat(ts_node_next_sibling(number_node), Equals(array_comma_node1));
@ -294,16 +346,16 @@ describe("Node", []() {
});
it("returns null when the node has no parent", [&]() {
AssertThat(ts_node_next_named_sibling(array_node).data, Equals<void *>(nullptr));
AssertThat(ts_node_prev_named_sibling(array_node).data, Equals<void *>(nullptr));
AssertThat(ts_node_next_named_sibling(root_node).data, Equals<void *>(nullptr));
AssertThat(ts_node_prev_named_sibling(root_node).data, Equals<void *>(nullptr));
});
});
describe("next_named_sibling(), prev_named_sibling()", [&]() {
it("returns the node's next and previous siblings", [&]() {
TSNode number_node = ts_node_named_child(array_node, 0);
TSNode false_node = ts_node_named_child(array_node, 1);
TSNode object_node = ts_node_named_child(array_node, 2);
TSNode number_node = ts_node_named_child(root_node, 0);
TSNode false_node = ts_node_named_child(root_node, 1);
TSNode object_node = ts_node_named_child(root_node, 2);
TSNode pair_node = ts_node_named_child(object_node, 0);
TSNode string_node = ts_node_named_child(pair_node, 0);
TSNode null_node = ts_node_named_child(pair_node, 1);
@ -317,22 +369,22 @@ describe("Node", []() {
});
it("returns null when the node has no parent", [&]() {
AssertThat(ts_node_next_named_sibling(array_node).data, Equals<void *>(nullptr));
AssertThat(ts_node_prev_named_sibling(array_node).data, Equals<void *>(nullptr));
AssertThat(ts_node_next_named_sibling(root_node).data, Equals<void *>(nullptr));
AssertThat(ts_node_prev_named_sibling(root_node).data, Equals<void *>(nullptr));
});
});
describe("named_descendant_for_char_range(start, end)", [&]() {
describe("when there is a leaf node that spans the given range exactly", [&]() {
it("returns that leaf node", [&]() {
TSNode leaf = ts_node_named_descendant_for_char_range(array_node, string_index, string_end_index - 1);
TSNode leaf = ts_node_named_descendant_for_char_range(root_node, string_index, string_end_index - 1);
AssertThat(ts_node_type(leaf, document), Equals("string"));
AssertThat(ts_node_start_byte(leaf), Equals(string_index));
AssertThat(ts_node_end_byte(leaf), Equals(string_end_index));
AssertThat(ts_node_start_point(leaf), Equals<TSPoint>({ 6, 4 }));
AssertThat(ts_node_end_point(leaf), Equals<TSPoint>({ 6, 7 }));
leaf = ts_node_named_descendant_for_char_range(array_node, number_index, number_end_index - 1);
leaf = ts_node_named_descendant_for_char_range(root_node, number_index, number_end_index - 1);
AssertThat(ts_node_type(leaf, document), Equals("number"));
AssertThat(ts_node_start_byte(leaf), Equals(number_index));
AssertThat(ts_node_end_byte(leaf), Equals(number_end_index));
@ -343,14 +395,14 @@ describe("Node", []() {
describe("when there is a leaf node that extends beyond the given range", [&]() {
it("returns that leaf node", [&]() {
TSNode leaf = ts_node_named_descendant_for_char_range(array_node, string_index, string_index + 1);
TSNode leaf = ts_node_named_descendant_for_char_range(root_node, string_index, string_index + 1);
AssertThat(ts_node_type(leaf, document), Equals("string"));
AssertThat(ts_node_start_byte(leaf), Equals(string_index));
AssertThat(ts_node_end_byte(leaf), Equals(string_end_index));
AssertThat(ts_node_start_point(leaf), Equals<TSPoint>({ 6, 4 }));
AssertThat(ts_node_end_point(leaf), Equals<TSPoint>({ 6, 7 }));
leaf = ts_node_named_descendant_for_char_range(array_node, string_index + 1, string_index + 2);
leaf = ts_node_named_descendant_for_char_range(root_node, string_index + 1, string_index + 2);
AssertThat(ts_node_type(leaf, document), Equals("string"));
AssertThat(ts_node_start_byte(leaf), Equals(string_index));
AssertThat(ts_node_end_byte(leaf), Equals(string_end_index));
@ -361,7 +413,7 @@ describe("Node", []() {
describe("when there is no leaf node that spans the given range", [&]() {
it("returns the smallest node that does span the range", [&]() {
TSNode pair_node = ts_node_named_descendant_for_char_range(array_node, string_index, string_index + 3);
TSNode pair_node = ts_node_named_descendant_for_char_range(root_node, string_index, string_index + 3);
AssertThat(ts_node_type(pair_node, document), Equals("pair"));
AssertThat(ts_node_start_byte(pair_node), Equals(string_index));
AssertThat(ts_node_end_byte(pair_node), Equals(null_end_index));
@ -370,7 +422,7 @@ describe("Node", []() {
});
it("does not return invisible nodes (repeats)", [&]() {
TSNode node = ts_node_named_descendant_for_char_range(array_node, number_end_index, number_end_index + 1);
TSNode node = ts_node_named_descendant_for_char_range(root_node, number_end_index, number_end_index + 1);
AssertThat(ts_node_type(node, document), Equals("array"));
AssertThat(ts_node_start_byte(node), Equals(array_index));
AssertThat(ts_node_end_byte(node), Equals(array_end_index));
@ -382,14 +434,14 @@ describe("Node", []() {
describe("descendant_for_char_range(start, end)", [&]() {
it("returns the smallest node that spans the given range", [&]() {
TSNode node1 = ts_node_descendant_for_char_range(array_node, colon_index, colon_index);
TSNode node1 = ts_node_descendant_for_char_range(root_node, colon_index, colon_index);
AssertThat(ts_node_type(node1, document), Equals(":"));
AssertThat(ts_node_start_byte(node1), Equals(colon_index));
AssertThat(ts_node_end_byte(node1), Equals(colon_index + 1));
AssertThat(ts_node_start_point(node1), Equals<TSPoint>({ 6, 7 }));
AssertThat(ts_node_end_point(node1), Equals<TSPoint>({ 6, 8 }));
TSNode node2 = ts_node_descendant_for_char_range(array_node, string_index + 2, string_index + 4);
TSNode node2 = ts_node_descendant_for_char_range(root_node, string_index + 2, string_index + 4);
AssertThat(ts_node_type(node2, document), Equals("pair"));
AssertThat(ts_node_start_byte(node2), Equals(string_index));
AssertThat(ts_node_end_byte(node2), Equals(null_end_index));
@ -402,12 +454,12 @@ describe("Node", []() {
it("returns the smallest concrete node that spans the given range", [&]() {
ts_document_set_input_string(document, "[\"αβγδ\", \"αβγδ\"]");
ts_document_parse(document);
TSNode array_node = ts_document_root_node(document);
TSNode root_node = ts_document_root_node(document);
TSNode node1 = ts_node_descendant_for_char_range(array_node, 7, 7);
TSNode node1 = ts_node_descendant_for_char_range(root_node, 7, 7);
AssertThat(ts_node_type(node1, document), Equals(","));
TSNode node2 = ts_node_descendant_for_byte_range(array_node, 6, 10);
TSNode node2 = ts_node_descendant_for_byte_range(root_node, 6, 10);
AssertThat(ts_node_type(node2, document), Equals("string"));
AssertThat(ts_node_start_byte(node2), Equals<size_t>(1));
AssertThat(ts_node_end_byte(node2), Equals<size_t>(11));
@ -416,14 +468,14 @@ describe("Node", []() {
describe("descendant_for_point_range(start, end)", [&]() {
it("returns the smallest concrete node that spans the given range", [&]() {
TSNode node1 = ts_node_descendant_for_point_range(array_node, {6, 7}, {6, 7});
TSNode node1 = ts_node_descendant_for_point_range(root_node, {6, 7}, {6, 7});
AssertThat(ts_node_type(node1, document), Equals(":"));
AssertThat(ts_node_start_byte(node1), Equals(colon_index));
AssertThat(ts_node_end_byte(node1), Equals(colon_index + 1));
AssertThat(ts_node_start_point(node1), Equals<TSPoint>({ 6, 7 }));
AssertThat(ts_node_end_point(node1), Equals<TSPoint>({ 6, 8 }));
TSNode node2 = ts_node_descendant_for_point_range(array_node, {6, 6}, {6, 8});
TSNode node2 = ts_node_descendant_for_point_range(root_node, {6, 6}, {6, 8});
AssertThat(ts_node_type(node2, document), Equals("pair"));
AssertThat(ts_node_start_byte(node2), Equals(string_index));
AssertThat(ts_node_end_byte(node2), Equals(null_end_index));