traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Make the empty chunk 2 bytes long, for UTF16 support

Browse source
This commit is contained in:
Max Brunsfeld 2018-06-25 17:46:23 -07:00
parent 4f5a87b952
commit 80cab8fd8a
2 changed files with 25 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

24
test/runtime/parser_test.cc
View file

@ -769,7 +769,7 @@ describe("Parser", [&]() {
});
});
describe("set_skipped_ranges", [&]() {
describe("set_included_ranges()", [&]() {
it("can parse code within a single range of a document", [&]() {
string source_code = "<span>hi</span><script>console.log('sup');</script>";
@ -874,6 +874,28 @@ describe("Parser", [&]() {
Equals<TSPoint>({0, static_cast<uint32_t>(source_code.find("<b>"))})
);
});
it("can handle errors at the ends of the nested UTF16 documents (regression)", [&]() {
u16string source_code = u"<script>a.</script>";
TSRange included_range = {
{0, static_cast<uint32_t>(2u * source_code.find(u"a."))},
{0, static_cast<uint32_t>(2u * source_code.find(u"</script"))},
2u * static_cast<uint32_t>(source_code.find(u"a.")),
2u * static_cast<uint32_t>(source_code.find(u"</script")),
};
ts_parser_set_logger(parser, stderr_logger_new(true));
ts_parser_set_included_ranges(parser, &included_range, 1);
ts_parser_set_language(parser, load_real_language("javascript"));
SpyInput input("", 3);
input.content.assign((const char *)source_code.c_str(), source_code.size() * 2);
input.encoding = TSInputEncodingUTF16;
tree = ts_parser_parse(parser, nullptr, input.input());
assert_root_node("(program (ERROR (identifier)))");
});
});
});