traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rework API completely

Browse source
This commit is contained in:
Max Brunsfeld 2018-05-10 22:22:37 -07:00
parent 33f7643040
commit e75ecd1bb1
31 changed files with 841 additions and 1075 deletions
Download patch file Download diff file Expand all files Collapse all files

490
test/runtime/document_test.cc
View file

@ -1,490 +0,0 @@
#include "test_helper.h"
#include "runtime/alloc.h"
#include "helpers/record_alloc.h"
#include "helpers/stream_methods.h"
#include "helpers/tree_helpers.h"
#include "helpers/point_helpers.h"
#include "helpers/spy_logger.h"
#include "helpers/stderr_logger.h"
#include "helpers/spy_input.h"
#include "helpers/load_language.h"
TSPoint point(size_t row, size_t column) {
return TSPoint{static_cast<uint32_t>(row), static_cast<uint32_t>(column)};
}
START_TEST
describe("Document", [&]() {
TSDocument *document;
TSNode root;
before_each([&]() {
record_alloc::start();
document = ts_document_new();
if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) {
ts_document_print_debugging_graphs(document, true);
}
});
after_each([&]() {
ts_document_free(document);
record_alloc::stop();
AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty());
});
auto assert_node_string_equals = [&](TSNode node, const string &expected) {
char *str = ts_node_string(node);
string actual(str);
ts_free(str);
AssertThat(actual, Equals(expected));
};
describe("set_input(input)", [&]() {
SpyInput *spy_input;
before_each([&]() {
spy_input = new SpyInput("{\"key\": [null, 2]}", 3);
ts_document_set_language(document, load_real_language("json"));
ts_document_set_input_string(document, "{\"key\": [1, 2]}");
ts_document_parse(document);
root = ts_document_root_node(document);
assert_node_string_equals(
root,
"(value (object (pair (string) (array (number) (number)))))");
});
after_each([&]() {
delete spy_input;
});
it("handles both UTF8 and UTF16 encodings", [&]() {
const char16_t content[] = u"[true, false]";
spy_input->content = string((const char *)content, sizeof(content));
spy_input->encoding = TSInputEncodingUTF16;
ts_document_set_input(document, spy_input->input());
ts_document_invalidate(document);
ts_document_parse(document);
root = ts_document_root_node(document);
assert_node_string_equals(
root,
"(value (array (true) (false)))");
});
it("handles truncated UTF16 data", [&]() {
const char content[1] = { '\0' };
spy_input->content = string(content, sizeof(content));
spy_input->encoding = TSInputEncodingUTF16;
ts_document_set_input(document, spy_input->input());
ts_document_invalidate(document);
ts_document_parse(document);
});
it("measures columns in bytes", [&]() {
const char16_t content[] = u"[true, false]";
spy_input->content = string((const char *)content, sizeof(content));
spy_input->encoding = TSInputEncodingUTF16;
TSInput input = spy_input->input();
ts_document_set_input(document, input);
ts_document_invalidate(document);
ts_document_parse(document);
root = ts_document_root_node(document);
AssertThat(ts_node_end_point(root), Equals<TSPoint>({0, 28}));
});
it("allows the input to be retrieved later", [&]() {
ts_document_set_input(document, spy_input->input());
AssertThat(ts_document_input(document).payload, Equals<void *>(spy_input));
AssertThat(ts_document_input(document).read, Equals(spy_input->input().read));
AssertThat(ts_document_input(document).seek, Equals(spy_input->input().seek));
});
it("does not assume that the document's text has changed", [&]() {
ts_document_set_input(document, spy_input->input());
AssertThat(ts_document_root_node(document), Equals<TSNode>(root));
AssertThat(ts_node_has_changes(root), IsFalse());
AssertThat(spy_input->strings_read(), IsEmpty());
});
it("reads text from the new input for future parses", [&]() {
ts_document_set_input(document, spy_input->input());
// Insert 'null', delete '1'.
TSInputEdit edit = {};
edit.start_point.column = edit.start_byte = strlen("{\"key\": [");
edit.extent_added.column = edit.bytes_added = 4;
edit.extent_removed.column = edit.bytes_removed = 1;
ts_document_edit(document, edit);
ts_document_parse(document);
TSNode new_root = ts_document_root_node(document);
assert_node_string_equals(
new_root,
"(value (object (pair (string) (array (null) (number)))))");
AssertThat(spy_input->strings_read(), Equals(vector<string>({" [null, 2" })));
});
it("allows setting input string with length", [&]() {
const char content[] = { '1' };
ts_document_set_input_string_with_length(document, content, 1);
ts_document_parse(document);
TSNode new_root = ts_document_root_node(document);
AssertThat(ts_node_end_byte(new_root), Equals<size_t>(1));
assert_node_string_equals(
new_root,
"(value (number))");
});
it("reads from the new input correctly when the old input was blank", [&]() {
ts_document_set_input_string(document, "");
ts_document_parse(document);
TSNode new_root = ts_document_root_node(document);
AssertThat(ts_node_end_byte(new_root), Equals<size_t>(0));
assert_node_string_equals(
new_root,
"(ERROR)");
ts_document_set_input_string(document, "1");
ts_document_parse(document);
new_root = ts_document_root_node(document);
AssertThat(ts_node_end_byte(new_root), Equals<size_t>(1));
assert_node_string_equals(
new_root,
"(value (number))");
});
});
describe("set_language(language)", [&]() {
before_each([&]() {
ts_document_set_input_string(document, "{\"key\": [1, 2]}\n");
});
it("uses the given language for future parses", [&]() {
ts_document_set_language(document, load_real_language("json"));
ts_document_parse(document);
root = ts_document_root_node(document);
assert_node_string_equals(
root,
"(value (object (pair (string) (array (number) (number)))))");
});
it("clears out any previous tree", [&]() {
ts_document_set_language(document, load_real_language("json"));
ts_document_parse(document);
ts_document_set_language(document, load_real_language("javascript"));
AssertThat(ts_document_root_node(document).subtree, Equals<void *>(nullptr));
ts_document_parse(document);
root = ts_document_root_node(document);
assert_node_string_equals(
root,
"(program (expression_statement "
"(object (pair (string) (array (number) (number))))))");
});
it("does not allow setting a language with a different version number", [&]() {
TSLanguage language = *load_real_language("json");
AssertThat(ts_language_version(&language), Equals<uint32_t>(TREE_SITTER_LANGUAGE_VERSION));
language.version++;
AssertThat(ts_language_version(&language), !Equals<uint32_t>(TREE_SITTER_LANGUAGE_VERSION));
ts_document_set_language(document, &language);
AssertThat(ts_document_language(document), Equals<const TSLanguage *>(nullptr));
});
});
describe("set_logger(TSLogger)", [&]() {
SpyLogger *logger;
before_each([&]() {
logger = new SpyLogger();
ts_document_set_language(document, load_real_language("json"));
ts_document_set_input_string(document, "[1, 2]");
});
after_each([&]() {
delete logger;
});
it("calls the debugger with a message for each parse action", [&]() {
ts_document_set_logger(document, logger->logger());
ts_document_parse(document);
AssertThat(logger->messages, Contains("new_parse"));
AssertThat(logger->messages, Contains("skip character:' '"));
AssertThat(logger->messages, Contains("consume character:'['"));
AssertThat(logger->messages, Contains("consume character:'1'"));
AssertThat(logger->messages, Contains("reduce sym:array, child_count:4"));
AssertThat(logger->messages, Contains("accept"));
});
it("allows the debugger to be retrieved later", [&]() {
ts_document_set_logger(document, logger->logger());
AssertThat(ts_document_logger(document).payload, Equals(logger));
});
describe("disabling debugging", [&]() {
before_each([&]() {
ts_document_set_logger(document, logger->logger());
ts_document_set_logger(document, {NULL, NULL});
});
it("does not call the debugger any more", [&]() {
ts_document_parse(document);
AssertThat(logger->messages, IsEmpty());
});
});
});
describe("parse_and_get_changed_ranges()", [&]() {
SpyInput *input;
before_each([&]() {
ts_document_set_language(document, load_real_language("javascript"));
input = new SpyInput("{a: null};\n", 3);
ts_document_set_input(document, input->input());
ts_document_parse(document);
assert_node_string_equals(
ts_document_root_node(document),
"(program (expression_statement (object (pair (property_identifier) (null)))))");
});
after_each([&]() {
delete input;
});
auto get_invalidated_ranges_for_edit = [&](std::function<TSInputEdit()> callback) -> vector<TSRange> {
TSInputEdit edit = callback();
ts_document_edit(document, edit);
TSRange *ranges;
uint32_t range_count = 0;
ts_document_parse_and_get_changed_ranges(document, &ranges, &range_count);
vector<TSRange> result;
for (size_t i = 0; i < range_count; i++) {
result.push_back(ranges[i]);
}
ts_free(ranges);
return result;
};
it("reports changes when one token has been updated", [&]() {
// Replace `null` with `nothing`
auto ranges = get_invalidated_ranges_for_edit([&]() {
return input->replace(input->content.find("ull"), 1, "othing");
});
AssertThat(ranges, Equals(vector<TSRange>({
TSRange{
point(0, input->content.find("nothing")),
point(0, input->content.find("}"))
},
})));
// Replace `nothing` with `null` again
ranges = get_invalidated_ranges_for_edit([&]() {
return input->undo();
});
AssertThat(ranges, Equals(vector<TSRange>({
TSRange{
point(0, input->content.find("null")),
point(0, input->content.find("}"))
},
})));
});
it("reports no changes when leading whitespace has changed (regression)", [&]() {
input->chars_per_chunk = 80;
// Insert leading whitespace
auto ranges = get_invalidated_ranges_for_edit([&]() {
return input->replace(0, 0, "\n");
});
assert_node_string_equals(
ts_document_root_node(document),
"(program (expression_statement (object (pair (property_identifier) (null)))))");
AssertThat(ranges, Equals(vector<TSRange>({})));
// Remove leading whitespace
ranges = get_invalidated_ranges_for_edit([&]() {
return input->undo();
});
assert_node_string_equals(
ts_document_root_node(document),
"(program (expression_statement (object (pair (property_identifier) (null)))))");
AssertThat(ranges, Equals(vector<TSRange>({})));
// Insert leading whitespace again
ranges = get_invalidated_ranges_for_edit([&]() {
return input->replace(0, 0, "\n");
});
assert_node_string_equals(
ts_document_root_node(document),
"(program (expression_statement (object (pair (property_identifier) (null)))))");
AssertThat(ranges, Equals(vector<TSRange>({})));
});
it("reports changes when tokens have been appended", [&]() {
// Add a second key-value pair
auto ranges = get_invalidated_ranges_for_edit([&]() {
return input->replace(input->content.find("}"), 0, ", b: false");
});
AssertThat(ranges, Equals(vector<TSRange>({
TSRange{
point(0, input->content.find(",")),
point(0, input->content.find("}"))
},
})));
// Add a third key-value pair in between the first two
ranges = get_invalidated_ranges_for_edit([&]() {
return input->replace(input->content.find(", b"), 0, ", c: 1");
});
assert_node_string_equals(
ts_document_root_node(document),
"(program (expression_statement (object "
"(pair (property_identifier) (null)) "
"(pair (property_identifier) (number)) "
"(pair (property_identifier) (false)))))");
AssertThat(ranges, Equals(vector<TSRange>({
TSRange{
point(0, input->content.find(", c")),
point(0, input->content.find(", b"))
},
})));
// Delete the middle pair.
ranges = get_invalidated_ranges_for_edit([&]() {
return input->undo();
});
assert_node_string_equals(
ts_document_root_node(document),
"(program (expression_statement (object "
"(pair (property_identifier) (null)) "
"(pair (property_identifier) (false)))))");
AssertThat(ranges, IsEmpty());
// Delete the second pair.
ranges = get_invalidated_ranges_for_edit([&]() {
return input->undo();
});
assert_node_string_equals(
ts_document_root_node(document),
"(program (expression_statement (object "
"(pair (property_identifier) (null)))))");
AssertThat(ranges, IsEmpty());
});
it("reports changes when trees have been wrapped", [&]() {
// Wrap the object in an assignment expression.
auto ranges = get_invalidated_ranges_for_edit([&]() {
return input->replace(input->content.find("null"), 0, "b === ");
});
assert_node_string_equals(
ts_document_root_node(document),
"(program (expression_statement (object "
"(pair (property_identifier) (binary_expression (identifier) (null))))))");
AssertThat(ranges, Equals(vector<TSRange>({
TSRange{
point(0, input->content.find("b ===")),
point(0, input->content.find("}"))
},
})));
});
});
describe("parse_with_options(options)", [&]() {
it("halts as soon as an error is found if the halt_on_error flag is set", [&]() {
string input_string = "[1, null, error, 3]";
ts_document_set_language(document, load_real_language("json"));
ts_document_set_input_string(document, input_string.c_str());
TSParseOptions options = {};
options.changed_ranges = nullptr;
options.halt_on_error = false;
ts_document_parse_with_options(document, options);
root = ts_document_root_node(document);
assert_node_string_equals(
root,
"(value (array (number) (null) (ERROR (UNEXPECTED 'e')) (number)))");
ts_document_invalidate(document);
options.halt_on_error = true;
ts_document_parse_with_options(document, options);
root = ts_document_root_node(document);
assert_node_string_equals(
root,
"(ERROR (number) (null))");
AssertThat(ts_node_end_byte(root), Equals(input_string.size()));
});
it("does not insert missing tokens if the halt_on_error flag is set", [&]() {
string input_string = "[1, null, 3";
ts_document_set_language(document, load_real_language("json"));
ts_document_set_input_string(document, input_string.c_str());
TSParseOptions options = {};
options.changed_ranges = nullptr;
options.halt_on_error = false;
ts_document_parse_with_options(document, options);
root = ts_document_root_node(document);
assert_node_string_equals(
root,
"(value (array (number) (null) (number) (MISSING)))");
ts_document_invalidate(document);
options.halt_on_error = true;
ts_document_parse_with_options(document, options);
root = ts_document_root_node(document);
assert_node_string_equals(
root,
"(ERROR (number) (null) (number))");
AssertThat(ts_node_end_byte(root), Equals(input_string.size()));
});
it("can parse valid code with the halt_on_error flag set", [&]() {
string input_string = "[1, null, 3]";
ts_document_set_language(document, load_real_language("json"));
ts_document_set_input_string(document, input_string.c_str());
TSParseOptions options = {};
options.changed_ranges = nullptr;
options.halt_on_error = true;
ts_document_parse_with_options(document, options);
root = ts_document_root_node(document);
assert_node_string_equals(
root,
"(value (array (number) (null) (number)))");
});
});
});
END_TEST

12
test/runtime/language_test.cc
View file

@ -28,13 +28,12 @@ describe("Language", []() {
}
})JSON");
TSDocument *document = ts_document_new();
TSParser *parser = ts_parser_new();
const TSLanguage *language = load_test_language("aliased_rules", compile_result);
ts_document_set_language(document, language);
ts_document_set_input_string(document, "b");
ts_document_parse(document);
ts_parser_set_language(parser, language);
TSTree *tree = ts_parser_parse_string(parser, nullptr, "b", 1);
TSNode root_node = ts_document_root_node(document);
TSNode root_node = ts_tree_root_node(tree);
char *string = ts_node_string(root_node);
AssertThat(string, Equals("(a (c))"));
@ -47,7 +46,8 @@ describe("Language", []() {
AssertThat(ts_language_symbol_type(language, aliased_symbol), Equals(TSSymbolTypeRegular));
ts_free(string);
ts_document_free(document);
ts_parser_delete(parser);
ts_tree_delete(tree);
});
});
});

56
test/runtime/node_test.cc
View file

@ -62,21 +62,22 @@ string grammar_with_aliases_and_extras = R"JSON({
})JSON";
describe("Node", [&]() {
TSDocument *document;
TSParser *parser;
TSTree *tree;
TSNode root_node;
before_each([&]() {
record_alloc::start();
document = ts_document_new();
ts_document_set_language(document, load_real_language("json"));
ts_document_set_input_string(document, json_string.c_str());
ts_document_parse(document);
root_node = ts_node_child(ts_document_root_node(document), 0);
parser = ts_parser_new();
ts_parser_set_language(parser, load_real_language("json"));
tree = ts_parser_parse_string(parser, nullptr, json_string.c_str(), json_string.size());
root_node = ts_node_child(ts_tree_root_node(tree), 0);
});
after_each([&]() {
ts_document_free(document);
ts_parser_delete(parser);
ts_tree_delete(tree);
record_alloc::stop();
AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty());
@ -157,16 +158,17 @@ describe("Node", [&]() {
AssertThat(ts_node_parent(number_node), Equals(root_node));
AssertThat(ts_node_parent(false_node), Equals(root_node));
AssertThat(ts_node_parent(object_node), Equals(root_node));
AssertThat(ts_node_parent(ts_document_root_node(document)).subtree, Equals<void *>(nullptr));
AssertThat(ts_node_parent(ts_tree_root_node(tree)).subtree, Equals<void *>(nullptr));
});
it("works correctly when the node contains aliased children and extras", [&]() {
TSCompileResult compile_result = ts_compile_grammar(grammar_with_aliases_and_extras.c_str());
const TSLanguage *language = load_test_language("aliases_and_extras", compile_result);
ts_document_set_language(document, language);
ts_document_set_input_string(document, "b ... b ... b");
ts_document_parse(document);
root_node = ts_document_root_node(document);
ts_parser_set_language(parser, language);
ts_tree_delete(tree);
tree = ts_parser_parse_string(parser, nullptr, "b ... b ... b", 13);
root_node = ts_tree_root_node(tree);
char *node_string = ts_node_string(root_node);
AssertThat(node_string, Equals("(a (b) (comment) (B) (comment) (b))"));
@ -179,7 +181,10 @@ describe("Node", [&]() {
AssertThat(ts_node_type(ts_node_named_child(root_node, 3)), Equals("comment"));
AssertThat(ts_node_type(ts_node_named_child(root_node, 4)), Equals("b"));
AssertThat(ts_node_symbol(ts_node_named_child(root_node, 0)), !Equals(ts_node_symbol(ts_node_named_child(root_node, 2))));
AssertThat(
ts_node_symbol(ts_node_named_child(root_node, 0)),
!Equals(ts_node_symbol(ts_node_named_child(root_node, 2)))
);
});
});
@ -323,7 +328,7 @@ describe("Node", [&]() {
AssertThat(ts_node_parent(child5), Equals(root_node));
AssertThat(ts_node_parent(child6), Equals(root_node));
AssertThat(ts_node_parent(child7), Equals(root_node));
AssertThat(ts_node_parent(ts_document_root_node(document)).subtree, Equals<void *>(nullptr));
AssertThat(ts_node_parent(ts_tree_root_node(tree)).subtree, Equals<void *>(nullptr));
});
});
@ -483,9 +488,10 @@ describe("Node", [&]() {
it("works in the presence of multi-byte characters", [&]() {
string input_string = "[\"αβγδ\", \"αβγδ\"]";
ts_document_set_input_string(document, input_string.c_str());
ts_document_parse(document);
TSNode root_node = ts_document_root_node(document);
ts_tree_delete(tree);
tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size());
TSNode root_node = ts_tree_root_node(tree);
uint32_t comma_position = input_string.find(",");
TSNode node1 = ts_node_descendant_for_byte_range(root_node, comma_position, comma_position);
@ -518,23 +524,23 @@ describe("Node", [&]() {
});
describe("TreeCursor", [&]() {
TSDocument *document;
TSParser *parser;
TSTree *tree;
TSTreeCursor *cursor;
before_each([&]() {
record_alloc::start();
document = ts_document_new();
ts_document_set_language(document, load_real_language("json"));
ts_document_set_input_string(document, json_string.c_str());
ts_document_parse(document);
cursor = ts_document_tree_cursor(document);
parser = ts_parser_new();
ts_parser_set_language(parser, load_real_language("json"));
tree = ts_parser_parse_string(parser, nullptr, json_string.c_str(), json_string.size());
cursor = ts_tree_cursor_new(tree);
});
after_each([&]() {
ts_tree_delete(tree);
ts_tree_cursor_delete(cursor);
ts_document_free(document);
ts_parser_delete(parser);
record_alloc::stop();
AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty());

266
test/runtime/parser_test.cc
View file

@ -1,17 +1,20 @@
#include "test_helper.h"
#include "runtime/alloc.h"
#include "runtime/language.h"
#include "helpers/record_alloc.h"
#include "helpers/spy_input.h"
#include "helpers/load_language.h"
#include "helpers/record_alloc.h"
#include "helpers/point_helpers.h"
#include "helpers/spy_logger.h"
#include "helpers/stderr_logger.h"
#include "helpers/dedent.h"
START_TEST
describe("Parser", [&]() {
TSDocument *document;
TSParser *parser;
TSTree *tree;
SpyInput *input;
TSNode root;
size_t chunk_size;
@ -21,14 +24,16 @@ describe("Parser", [&]() {
chunk_size = 3;
input = nullptr;
document = ts_document_new();
tree = nullptr;
parser = ts_parser_new();
if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) {
ts_document_print_debugging_graphs(document, true);
ts_parser_print_debugging_graphs(parser, true);
}
});
after_each([&]() {
if (document) ts_document_free(document);
if (parser) ts_parser_delete(parser);
if (tree) ts_tree_delete(tree);
if (input) delete input;
record_alloc::stop();
@ -37,10 +42,8 @@ describe("Parser", [&]() {
auto set_text = [&](string text) {
input = new SpyInput(text, chunk_size);
ts_document_set_input(document, input->input());
ts_document_parse(document);
root = ts_document_root_node(document);
tree = ts_parser_parse(parser, nullptr, input->input());
root = ts_tree_root_node(tree);
AssertThat(ts_node_end_byte(root), Equals(text.size()));
input->clear();
};
@ -48,10 +51,13 @@ describe("Parser", [&]() {
auto replace_text = [&](size_t position, size_t length, string new_text) {
size_t prev_size = ts_node_end_byte(root);
ts_document_edit(document, input->replace(position, length, new_text));
ts_document_parse(document);
TSInputEdit edit = input->replace(position, length, new_text);
ts_tree_edit(tree, &edit);
TSTree *new_tree = ts_parser_parse(parser, tree, input->input());
ts_tree_delete(tree);
tree = new_tree;
root = ts_document_root_node(document);
root = ts_tree_root_node(tree);
size_t new_size = ts_node_end_byte(root);
AssertThat(new_size, Equals(prev_size - length + new_text.size()));
};
@ -65,12 +71,15 @@ describe("Parser", [&]() {
};
auto undo = [&]() {
ts_document_edit(document, input->undo());
ts_document_parse(document);
TSInputEdit edit = input->undo();
ts_tree_edit(tree, &edit);
TSTree *new_tree = ts_parser_parse(parser, tree, input->input());
ts_tree_delete(tree);
tree = new_tree;
};
auto assert_root_node = [&](const string &expected) {
TSNode node = ts_document_root_node(document);
TSNode node = ts_tree_root_node(tree);
char *node_string = ts_node_string(node);
string actual(node_string);
ts_free(node_string);
@ -86,11 +95,9 @@ describe("Parser", [&]() {
describe("handling errors", [&]() {
describe("when there is an invalid substring right before a valid token", [&]() {
it("computes the error node's size and position correctly", [&]() {
ts_document_set_language(document, load_real_language("json"));
ts_parser_set_language(parser, load_real_language("json"));
set_text(" [123, @@@@@, true]");
assert_root_node(
"(value (array (number) (ERROR (UNEXPECTED '@')) (true)))");
assert_root_node("(value (array (number) (ERROR (UNEXPECTED '@')) (true)))");
TSNode error = ts_node_named_child(ts_node_child(root, 0), 1);
AssertThat(ts_node_type(error), Equals("ERROR"));
@ -111,7 +118,7 @@ describe("Parser", [&]() {
describe("when there is an unexpected string in the middle of a token", [&]() {
it("computes the error node's size and position correctly", [&]() {
ts_document_set_language(document, load_real_language("json"));
ts_parser_set_language(parser, load_real_language("json"));
set_text(" [123, faaaaalse, true]");
assert_root_node(
@ -138,11 +145,10 @@ describe("Parser", [&]() {
describe("when there is one unexpected token between two valid tokens", [&]() {
it("computes the error node's size and position correctly", [&]() {
ts_document_set_language(document, load_real_language("json"));
ts_parser_set_language(parser, load_real_language("json"));
set_text(" [123, true false, true]");
assert_root_node(
"(value (array (number) (true) (ERROR (false)) (true)))");
assert_root_node("(value (array (number) (true) (ERROR (false)) (true)))");
TSNode error = ts_node_named_child(ts_node_child(root, 0), 2);
AssertThat(ts_node_type(error), Equals("ERROR"));
@ -157,26 +163,23 @@ describe("Parser", [&]() {
describe("when there is an unexpected string at the end of a token", [&]() {
it("computes the error's size and position correctly", [&]() {
ts_document_set_language(document, load_real_language("json"));
ts_parser_set_language(parser, load_real_language("json"));
set_text(" [123, \"hi\n, true]");
assert_root_node(
"(value (array (number) (ERROR (UNEXPECTED '\\n')) (true)))");
assert_root_node("(value (array (number) (ERROR (UNEXPECTED '\\n')) (true)))");
});
});
describe("when there is an unterminated error", [&]() {
it("maintains a consistent tree", [&]() {
ts_document_set_language(document, load_real_language("javascript"));
ts_parser_set_language(parser, load_real_language("javascript"));
set_text("a; ' this string never ends");
assert_root_node(
"(program (expression_statement (identifier)) (ERROR (UNEXPECTED EOF)))");
assert_root_node("(program (expression_statement (identifier)) (ERROR (UNEXPECTED EOF)))");
});
});
describe("when there are extra tokens at the end of the viable prefix", [&]() {
it("does not include them in the error node", [&]() {
ts_document_set_language(document, load_real_language("javascript"));
ts_parser_set_language(parser, load_real_language("javascript"));
set_text(
"var x;\n"
"\n"
@ -196,20 +199,64 @@ describe("Parser", [&]() {
char *string = (char *)malloc(1);
string[0] = '\xdf';
ts_document_set_language(document, load_real_language("json"));
ts_document_set_input_string_with_length(document, string, 1);
ts_document_parse(document);
ts_parser_set_language(parser, load_real_language("json"));
tree = ts_parser_parse_string(parser, nullptr, string, 1);
free(string);
assert_root_node("(ERROR (UNEXPECTED INVALID))");
});
describe("when halt_on_error is set to true", [&]() {
it("halts as soon as an error is found if the halt_on_error flag is set", [&]() {
string input_string = "[1, null, error, 3]";
ts_parser_set_language(parser, load_real_language("json"));
tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size());
root = ts_tree_root_node(tree);
assert_root_node("(value (array (number) (null) (ERROR (UNEXPECTED 'e')) (number)))");
ts_parser_halt_on_error(parser, true);
ts_tree_delete(tree);
tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size());
root = ts_tree_root_node(tree);
assert_root_node("(ERROR (number) (null))");
AssertThat(ts_node_end_byte(root), Equals(input_string.size()));
});
it("does not insert missing tokens if the halt_on_error flag is set", [&]() {
string input_string = "[1, null, 3";
ts_parser_set_language(parser, load_real_language("json"));
tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size());
root = ts_tree_root_node(tree);
assert_root_node("(value (array (number) (null) (number) (MISSING)))");
ts_parser_halt_on_error(parser, true);
ts_tree_delete(tree);
tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size());
root = ts_tree_root_node(tree);
assert_root_node("(ERROR (number) (null) (number))");
AssertThat(ts_node_end_byte(root), Equals(input_string.size()));
});
it("can parse valid code with the halt_on_error flag set", [&]() {
string input_string = "[1, null, 3]";
ts_parser_set_language(parser, load_real_language("json"));
ts_parser_halt_on_error(parser, true);
tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size());
root = ts_tree_root_node(tree);
assert_root_node("(value (array (number) (null) (number)))");
});
});
});
describe("editing", [&]() {
describe("creating new tokens near the end of the input", [&]() {
it("updates the parse tree and re-reads only the changed portion of the text", [&]() {
ts_document_set_language(document, load_real_language("javascript"));
ts_parser_set_language(parser, load_real_language("javascript"));
set_text("x * (100 + abc);");
assert_root_node(
@ -242,7 +289,7 @@ describe("Parser", [&]() {
it("updates the parse tree and re-reads only the changed portion of the input", [&]() {
chunk_size = 2;
ts_document_set_language(document, load_real_language("javascript"));
ts_parser_set_language(parser, load_real_language("javascript"));
set_text("123 + 456 * (10 + x);");
assert_root_node(
@ -268,7 +315,7 @@ describe("Parser", [&]() {
describe("introducing an error", [&]() {
it("gives the error the right size", [&]() {
ts_document_set_language(document, load_real_language("javascript"));
ts_parser_set_language(parser, load_real_language("javascript"));
set_text("var x = y;");
assert_root_node(
@ -291,7 +338,7 @@ describe("Parser", [&]() {
describe("into the middle of an existing token", [&]() {
it("updates the parse tree", [&]() {
ts_document_set_language(document, load_real_language("javascript"));
ts_parser_set_language(parser, load_real_language("javascript"));
set_text("abc * 123;");
assert_root_node(
@ -310,7 +357,7 @@ describe("Parser", [&]() {
describe("at the end of an existing token", [&]() {
it("updates the parse tree", [&]() {
ts_document_set_language(document, load_real_language("javascript"));
ts_parser_set_language(parser, load_real_language("javascript"));
set_text("abc * 123;");
assert_root_node(
@ -329,7 +376,7 @@ describe("Parser", [&]() {
describe("inserting text into a node containing a extra token", [&]() {
it("updates the parse tree", [&]() {
ts_document_set_language(document, load_real_language("javascript"));
ts_parser_set_language(parser, load_real_language("javascript"));
set_text("123 *\n"
"// a-comment\n"
"abc;");
@ -356,7 +403,7 @@ describe("Parser", [&]() {
describe("when a critical token is removed", [&]() {
it("updates the parse tree, creating an error", [&]() {
ts_document_set_language(document, load_real_language("javascript"));
ts_parser_set_language(parser, load_real_language("javascript"));
set_text("123 * 456; 789 * 123;");
assert_root_node(
@ -376,7 +423,7 @@ describe("Parser", [&]() {
describe("with external tokens", [&]() {
it("maintains the external scanner's state during incremental parsing", [&]() {
ts_document_set_language(document, load_real_language("python"));
ts_parser_set_language(parser, load_real_language("python"));
string text = dedent(R"PYTHON(
if a:
print b
@ -404,7 +451,7 @@ describe("Parser", [&]() {
});
it("does not try to reuse nodes that are within the edited region", [&]() {
ts_document_set_language(document, load_real_language("javascript"));
ts_parser_set_language(parser, load_real_language("javascript"));
set_text("{ x: (b.c) };");
assert_root_node(
@ -417,23 +464,12 @@ describe("Parser", [&]() {
"(program (expression_statement (object (pair "
"(property_identifier) (member_expression (identifier) (property_identifier))))))");
});
it("updates the document's parse count", [&]() {
ts_document_set_language(document, load_real_language("javascript"));
AssertThat(ts_document_parse_count(document), Equals<size_t>(0));
set_text("{ x: (b.c) };");
AssertThat(ts_document_parse_count(document), Equals<size_t>(1));
insert_text(strlen("{ x"), "yz");
AssertThat(ts_document_parse_count(document), Equals<size_t>(2));
});
});
describe("lexing", [&]() {
describe("handling tokens containing wildcard patterns (e.g. comments)", [&]() {
it("terminates them at the end of the document", [&]() {
ts_document_set_language(document, load_real_language("javascript"));
it("terminates them at the end of the string", [&]() {
ts_parser_set_language(parser, load_real_language("javascript"));
set_text("x; // this is a comment");
assert_root_node(
@ -448,7 +484,7 @@ describe("Parser", [&]() {
it("recognizes UTF8 characters as single characters", [&]() {
// 'ΩΩΩ — ΔΔ';
ts_document_set_language(document, load_real_language("javascript"));
ts_parser_set_language(parser, load_real_language("javascript"));
set_text("'\u03A9\u03A9\u03A9 \u2014 \u0394\u0394';");
assert_root_node(
@ -460,14 +496,120 @@ describe("Parser", [&]() {
it("handles non-UTF8 characters", [&]() {
const char *string = "cons\xeb\x00e=ls\x83l6hi');\x0a";
ts_document_set_language(document, load_real_language("javascript"));
ts_document_set_input_string(document, string);
ts_document_parse(document);
TSNode root = ts_document_root_node(document);
ts_parser_set_language(parser, load_real_language("javascript"));
tree = ts_parser_parse_string(parser, nullptr, string, strlen(string));
TSNode root = ts_tree_root_node(tree);
AssertThat(ts_node_end_byte(root), Equals(strlen(string)));
});
});
describe("handling TSInputs", [&]() {
SpyInput *spy_input;
before_each([&]() {
spy_input = new SpyInput("{\"key\": [null, 2]}", 3);
ts_parser_set_language(parser, load_real_language("json"));
});
after_each([&]() {
delete spy_input;
});
it("handles UTF16 encodings", [&]() {
const char16_t content[] = u"[true, false]";
spy_input->content = string((const char *)content, sizeof(content));
spy_input->encoding = TSInputEncodingUTF16;
tree = ts_parser_parse(parser, nullptr, spy_input->input());
root = ts_tree_root_node(tree);
assert_root_node(
"(value (array (true) (false)))");
});
it("handles truncated UTF16 data", [&]() {
const char content[1] = { '\0' };
spy_input->content = string(content, sizeof(content));
spy_input->encoding = TSInputEncodingUTF16;
tree = ts_parser_parse(parser, nullptr, spy_input->input());
});
it("measures columns in bytes", [&]() {
const char16_t content[] = u"[true, false]";
spy_input->content = string((const char *)content, sizeof(content));
spy_input->encoding = TSInputEncodingUTF16;
tree = ts_parser_parse(parser, nullptr, spy_input->input());
root = ts_tree_root_node(tree);
AssertThat(ts_node_end_point(root), Equals<TSPoint>({0, 28}));
});
});
describe("set_language(language)", [&]() {
string input_string = "{\"key\": [1, 2]}\n";
it("uses the given language for future parses", [&]() {
ts_parser_set_language(parser, load_real_language("json"));
tree = ts_parser_parse_string(parser, nullptr, input_string.c_str(), input_string.size());
root = ts_tree_root_node(tree);
assert_root_node(
"(value (object (pair (string) (array (number) (number)))))");
});
it("does not allow setting a language with a different version number", [&]() {
TSLanguage language = *load_real_language("json");
AssertThat(ts_language_version(&language), Equals<uint32_t>(TREE_SITTER_LANGUAGE_VERSION));
language.version++;
AssertThat(ts_language_version(&language), !Equals<uint32_t>(TREE_SITTER_LANGUAGE_VERSION));
AssertThat(ts_parser_set_language(parser, &language), IsFalse());
AssertThat(ts_parser_language(parser), Equals<const TSLanguage *>(nullptr));
});
});
describe("set_logger(TSLogger)", [&]() {
SpyLogger *logger;
before_each([&]() {
logger = new SpyLogger();
ts_parser_set_language(parser, load_real_language("json"));
});
after_each([&]() {
delete logger;
});
it("calls the debugger with a message for each parse action", [&]() {
ts_parser_set_logger(parser, logger->logger());
tree = ts_parser_parse_string(parser, nullptr, "[ 1, 2, 3 ]", 11);
AssertThat(logger->messages, Contains("new_parse"));
AssertThat(logger->messages, Contains("skip character:' '"));
AssertThat(logger->messages, Contains("consume character:'['"));
AssertThat(logger->messages, Contains("consume character:'1'"));
AssertThat(logger->messages, Contains("reduce sym:array, child_count:4"));
AssertThat(logger->messages, Contains("accept"));
});
it("allows the debugger to be retrieved later", [&]() {
ts_parser_set_logger(parser, logger->logger());
AssertThat(ts_parser_logger(parser).payload, Equals(logger));
});
describe("disabling debugging", [&]() {
before_each([&]() {
ts_parser_set_logger(parser, logger->logger());
ts_parser_set_logger(parser, {NULL, NULL});
});
it("does not call the debugger any more", [&]() {
tree = ts_parser_parse_string(parser, nullptr, "{}", 2);
AssertThat(logger->messages, IsEmpty());
});
});
});
});
END_TEST

2
test/runtime/stack_test.cc
View file

@ -76,7 +76,7 @@ describe("Stack", [&]() {
before_each([&]() {
record_alloc::start();
ts_subtree_pool_init(&pool);
pool = ts_subtree_pool_new(10);
stack = ts_stack_new(&pool);
TSLanguage dummy_language;

2
test/runtime/subtree_test.cc
View file

@ -41,7 +41,7 @@ describe("Subtree", []() {
SubtreePool pool;
before_each([&]() {
ts_subtree_pool_init(&pool);
pool = ts_subtree_pool_new(10);
});
after_each([&]() {

200
test/runtime/tree_test.cc Normal file
View file

@ -0,0 +1,200 @@
#include "test_helper.h"
#include "runtime/alloc.h"
#include "helpers/record_alloc.h"
#include "helpers/stream_methods.h"
#include "helpers/tree_helpers.h"
#include "helpers/point_helpers.h"
#include "helpers/spy_logger.h"
#include "helpers/stderr_logger.h"
#include "helpers/spy_input.h"
#include "helpers/load_language.h"
TSPoint point(uint32_t row, uint32_t column) {
TSPoint result = {row, column};
return result;
}
START_TEST
describe("Tree", [&]() {
TSParser *parser;
SpyInput *input;
TSTree *tree;
before_each([&]() {
parser = ts_parser_new();
});
after_each([&]() {
ts_parser_delete(parser);
});
auto assert_root_node = [&](const string &expected) {
TSNode node = ts_tree_root_node(tree);
char *node_string = ts_node_string(node);
string actual(node_string);
ts_free(node_string);
AssertThat(actual, Equals(expected));
};
describe("get_changed_ranges()", [&]() {
before_each([&]() {
ts_parser_set_language(parser, load_real_language("javascript"));
input = new SpyInput("{a: null};\n", 3);
tree = ts_parser_parse(parser, nullptr, input->input());
assert_root_node(
"(program (expression_statement (object (pair (property_identifier) (null)))))"
);
});
after_each([&]() {
ts_tree_delete(tree);
delete input;
});
auto get_changed_ranges_for_edit = [&](function<TSInputEdit()> fn) -> vector<TSRange> {
TSInputEdit edit = fn();
ts_tree_edit(tree, &edit);
uint32_t range_count = 0;
TSTree *new_tree = ts_parser_parse(parser, tree, input->input());
TSRange *ranges = ts_tree_get_changed_ranges(tree, new_tree, &range_count);
ts_tree_delete(tree);
tree = new_tree;
vector<TSRange> result;
for (size_t i = 0; i < range_count; i++) {
result.push_back(ranges[i]);
}
ts_free(ranges);
return result;
};
it("reports changes when one token has been updated", [&]() {
// Replace `null` with `nothing`
auto ranges = get_changed_ranges_for_edit([&]() {
return input->replace(input->content.find("ull"), 1, "othing");
});
AssertThat(ranges, Equals(vector<TSRange>({
TSRange{
point(0, input->content.find("nothing")),
point(0, input->content.find("}"))
},
})));
// Replace `nothing` with `null` again
ranges = get_changed_ranges_for_edit([&]() {
return input->undo();
});
AssertThat(ranges, Equals(vector<TSRange>({
TSRange{
point(0, input->content.find("null")),
point(0, input->content.find("}"))
},
})));
});
it("reports no changes when leading whitespace has changed (regression)", [&]() {
input->chars_per_chunk = 80;
// Insert leading whitespace
auto ranges = get_changed_ranges_for_edit([&]() {
return input->replace(0, 0, "\n");
});
assert_root_node(
"(program (expression_statement (object (pair (property_identifier) (null)))))"
);
AssertThat(ranges, IsEmpty());
// Remove leading whitespace
ranges = get_changed_ranges_for_edit([&]() {
return input->undo();
});
assert_root_node(
"(program (expression_statement (object (pair (property_identifier) (null)))))"
);
AssertThat(ranges, IsEmpty());
// Insert leading whitespace again
ranges = get_changed_ranges_for_edit([&]() {
return input->replace(0, 0, "\n");
});
assert_root_node(
"(program (expression_statement (object (pair (property_identifier) (null)))))"
);
AssertThat(ranges, IsEmpty());
});
it("reports changes when tokens have been appended", [&]() {
// Add a second key-value pair
auto ranges = get_changed_ranges_for_edit([&]() {
return input->replace(input->content.find("}"), 0, ", b: false");
});
AssertThat(ranges, Equals(vector<TSRange>({
TSRange{
point(0, input->content.find(",")),
point(0, input->content.find("}"))
},
})));
// Add a third key-value pair in between the first two
ranges = get_changed_ranges_for_edit([&]() {
return input->replace(input->content.find(", b"), 0, ", c: 1");
});
assert_root_node(
"(program (expression_statement (object "
"(pair (property_identifier) (null)) "
"(pair (property_identifier) (number)) "
"(pair (property_identifier) (false)))))"
);
AssertThat(ranges, Equals(vector<TSRange>({
TSRange{
point(0, input->content.find(", c")),
point(0, input->content.find(", b"))
},
})));
// Delete the middle pair.
ranges = get_changed_ranges_for_edit([&]() {
return input->undo();
});
assert_root_node(
"(program (expression_statement (object "
"(pair (property_identifier) (null)) "
"(pair (property_identifier) (false)))))"
);
AssertThat(ranges, IsEmpty());
// Delete the second pair.
ranges = get_changed_ranges_for_edit([&]() {
return input->undo();
});
assert_root_node(
"(program (expression_statement (object "
"(pair (property_identifier) (null)))))"
);
AssertThat(ranges, IsEmpty());
});
it("reports changes when trees have been wrapped", [&]() {
// Wrap the object in an assignment expression.
auto ranges = get_changed_ranges_for_edit([&]() {
return input->replace(input->content.find("null"), 0, "b === ");
});
assert_root_node(
"(program (expression_statement (object "
"(pair (property_identifier) (binary_expression (identifier) (null))))))"
);
AssertThat(ranges, Equals(vector<TSRange>({
TSRange{
point(0, input->content.find("b ===")),
point(0, input->content.find("}"))
},
})));
});
});
});
END_TEST