traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix end positions of tokens at the end of included ranges

Browse source 
Co-Authored-By: Ashi Krishnan <queerviolet@github.com>
This commit is contained in:
Max Brunsfeld 2018-07-09 10:23:25 -07:00
parent 3169620ce4
commit 83f88164aa
2 changed files with 35 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

12
src/runtime/lexer.c
View file

@ -97,7 +97,17 @@ static void ts_lexer__advance(void *payload, bool skip) {
static void ts_lexer__mark_end(void *payload) {
Lexer *self = (Lexer *)payload;
self->token_end_position = self->current_position;
TSRange *current_included_range = &self->included_ranges[self->current_included_range_index];
if (self->current_included_range_index > 0 &&
self->current_position.bytes == current_included_range->start_byte) {
TSRange *previous_included_range = current_included_range - 1;
self->token_end_position = (Length) {
previous_included_range->end_byte,
previous_included_range->end_point,
};
} else {
self->token_end_position = self->current_position;
}
}
static uint32_t ts_lexer__get_column(void *payload) {

25
test/runtime/parser_test.cc
View file

@ -862,8 +862,9 @@ describe("Parser", [&]() {
"(end_tag (tag_name))))");
root_node = ts_tree_root_node(tree);
TSNode hello_text_node = ts_node_child(ts_node_child(root_node, 0), 1);
TSNode div_element_node = ts_node_child(root_node, 0);
TSNode hello_text_node = ts_node_child(div_element_node, 1);
AssertThat(ts_node_type(hello_text_node), Equals("text"));
AssertThat(
ts_node_start_point(hello_text_node),
@ -873,6 +874,28 @@ describe("Parser", [&]() {
ts_node_end_point(hello_text_node),
Equals<TSPoint>({0, static_cast<uint32_t>(source_code.find("<b>"))})
);
TSNode b_start_tag_node = ts_node_child(ts_node_child(div_element_node, 2), 0);
AssertThat(ts_node_type(b_start_tag_node), Equals("start_tag"));
AssertThat(
ts_node_start_point(b_start_tag_node),
Equals<TSPoint>({0, static_cast<uint32_t>(source_code.find("<b>"))})
);
AssertThat(
ts_node_end_point(b_start_tag_node),
Equals<TSPoint>({0, static_cast<uint32_t>(source_code.find("${now()}"))})
);
TSNode b_end_tag_node = ts_node_child(ts_node_child(div_element_node, 2), 1);
AssertThat(ts_node_type(b_end_tag_node), Equals("end_tag"));
AssertThat(
ts_node_start_point(b_end_tag_node),
Equals<TSPoint>({0, static_cast<uint32_t>(source_code.find("</b>"))})
);
AssertThat(
ts_node_end_point(b_end_tag_node),
Equals<TSPoint>({0, static_cast<uint32_t>(source_code.find(".</div>"))})
);
});
it("can handle errors at the ends of the nested UTF16 documents (regression)", [&]() {