traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Restore external scanner's state only after repositioning lexer

Browse source 
Also, properly identify the leaf node with the external token state
This commit is contained in:
Max Brunsfeld 2016-12-21 13:59:56 -08:00
parent 3706678b89
commit 2fa7b453c8
5 changed files with 38 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

16
src/runtime/lexer.c
View file

@ -88,6 +88,8 @@ void ts_lexer_init(Lexer *self) {
.payload = NULL,
.log = NULL
},
.needs_to_restore_external_scanner = false,
.last_external_token_end_byte = 0,
};
ts_lexer_reset(self, length_zero());
}
@ -110,12 +112,22 @@ static inline void ts_lexer__reset(Lexer *self, Length position) {
void ts_lexer_set_input(Lexer *self, TSInput input) {
self->input = input;
ts_lexer__reset(self, length_zero());
self->needs_to_restore_external_scanner = false;
self->last_external_token_end_byte = 0;
}
void ts_lexer_reset(Lexer *self, Length position) {
if (!length_eq(position, self->current_position))
if (position.bytes > self->current_position.bytes) {
self->needs_to_restore_external_scanner = true;
self->last_external_token_end_byte = 0;
ts_lexer__reset(self, position);
return;
} else if (position.bytes < self->current_position.bytes) {
if (position.bytes < self->last_external_token_end_byte) {
self->needs_to_restore_external_scanner = true;
self->last_external_token_end_byte = 0;
}
ts_lexer__reset(self, position);
}
}
void ts_lexer_start(Lexer *self) {