Fix suppression of empty tokens during error handling at included range boundaries
This commit is contained in:
parent
d07f864815
commit
efd22e452b
2 changed files with 14 additions and 5 deletions
|
|
@ -346,6 +346,13 @@ void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte) {
|
|||
ts_lexer__mark_end(&self->data);
|
||||
}
|
||||
|
||||
// If the token ended at an included range boundary, then its end position
|
||||
// will have been reset to the end of the preceding range. Reset the start
|
||||
// position to match.
|
||||
if (self->token_end_position.bytes < self->token_start_position.bytes) {
|
||||
self->token_start_position = self->token_end_position;
|
||||
}
|
||||
|
||||
uint32_t current_lookahead_end_byte = self->current_position.bytes + 1;
|
||||
|
||||
// In order to determine that a byte sequence is invalid UTF8 or UTF16,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue