traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Handle input chunks that end within multi-byte characters

Browse source
This commit is contained in:
Max Brunsfeld 2018-08-02 15:30:40 -07:00
parent 126f84aa73
commit acc937b7d7
10 changed files with 64 additions and 116 deletions
Download patch file Download diff file Expand all files Collapse all files

15
test/runtime/parser_test.cc
View file

@ -544,6 +544,21 @@ describe("Parser", [&]() {
root = ts_tree_root_node(tree);
AssertThat(ts_node_end_point(root), Equals<TSPoint>({0, 28}));
});
it("handles input chunks that end in the middle of multi-byte characters", [&]() {
ts_parser_set_language(parser, load_real_language("c"));
spy_input->content = "A b = {'👍','👍'};";
spy_input->chars_per_chunk = 4;
tree = ts_parser_parse(parser, nullptr, spy_input->input());
root = ts_tree_root_node(tree);
assert_root_node(
"(translation_unit (declaration "
"(type_identifier) "
"(init_declarator "
"(identifier) "
"(initializer_list (char_literal) (char_literal)))))");
});
});
describe("set_language(language)", [&]() {

5
test/runtime/tree_test.cc
View file

@ -11,7 +11,6 @@
#include "helpers/load_language.h"
#include "helpers/random_helpers.h"
#include "helpers/read_test_entries.h"
#include "helpers/encoding_helpers.h"
#include "helpers/tree_helpers.h"
TSPoint point(uint32_t row, uint32_t column) {
@ -71,8 +70,8 @@ describe("Tree", [&]() {
for (unsigned j = 0; j < 10; j++) {
random.sleep_some();
size_t edit_position = random(utf8_char_count(input->content));
size_t deletion_size = random(utf8_char_count(input->content) - edit_position);
size_t edit_position = random(input->content.size());
size_t deletion_size = random(input->content.size() - edit_position);
string inserted_text = random.words(random(4) + 1);
TSInputEdit edit = input->replace(edit_position, deletion_size, inserted_text);