diff --git a/src/runtime/lexer.c b/src/runtime/lexer.c index ee4e4c7c..fcd1fd3f 100644 --- a/src/runtime/lexer.c +++ b/src/runtime/lexer.c @@ -97,7 +97,17 @@ static void ts_lexer__advance(void *payload, bool skip) { static void ts_lexer__mark_end(void *payload) { Lexer *self = (Lexer *)payload; - self->token_end_position = self->current_position; + TSRange *current_included_range = &self->included_ranges[self->current_included_range_index]; + if (self->current_included_range_index > 0 && + self->current_position.bytes == current_included_range->start_byte) { + TSRange *previous_included_range = current_included_range - 1; + self->token_end_position = (Length) { + previous_included_range->end_byte, + previous_included_range->end_point, + }; + } else { + self->token_end_position = self->current_position; + } } static uint32_t ts_lexer__get_column(void *payload) { diff --git a/test/runtime/parser_test.cc b/test/runtime/parser_test.cc index 182be18b..c4586cf7 100644 --- a/test/runtime/parser_test.cc +++ b/test/runtime/parser_test.cc @@ -862,8 +862,9 @@ describe("Parser", [&]() { "(end_tag (tag_name))))"); root_node = ts_tree_root_node(tree); - TSNode hello_text_node = ts_node_child(ts_node_child(root_node, 0), 1); + TSNode div_element_node = ts_node_child(root_node, 0); + TSNode hello_text_node = ts_node_child(div_element_node, 1); AssertThat(ts_node_type(hello_text_node), Equals("text")); AssertThat( ts_node_start_point(hello_text_node), @@ -873,6 +874,28 @@ describe("Parser", [&]() { ts_node_end_point(hello_text_node), Equals({0, static_cast(source_code.find(""))}) ); + + TSNode b_start_tag_node = ts_node_child(ts_node_child(div_element_node, 2), 0); + AssertThat(ts_node_type(b_start_tag_node), Equals("start_tag")); + AssertThat( + ts_node_start_point(b_start_tag_node), + Equals({0, static_cast(source_code.find(""))}) + ); + AssertThat( + ts_node_end_point(b_start_tag_node), + Equals({0, static_cast(source_code.find("${now()}"))}) + ); + + TSNode b_end_tag_node = ts_node_child(ts_node_child(div_element_node, 2), 1); + AssertThat(ts_node_type(b_end_tag_node), Equals("end_tag")); + AssertThat( + ts_node_start_point(b_end_tag_node), + Equals({0, static_cast(source_code.find(""))}) + ); + AssertThat( + ts_node_end_point(b_end_tag_node), + Equals({0, static_cast(source_code.find("."))}) + ); }); it("can handle errors at the ends of the nested UTF16 documents (regression)", [&]() {