traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Make the empty chunk 2 bytes long, for UTF16 support

Browse source
This commit is contained in:
Max Brunsfeld 2018-06-25 17:46:23 -07:00
parent 4f5a87b952
commit 80cab8fd8a
2 changed files with 25 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/runtime/lexer.c
View file

@ -14,7 +14,7 @@
#define LOG_CHARACTER(message, character) \
LOG(character < 255 ? message " character:'%c'" : message " character:%d", character)
static const char empty_chunk[2] = { 0, 0 };
static const char empty_chunk[3] = { 0, 0 };
static void ts_lexer__get_chunk(Lexer *self) {
self->chunk_start = self->current_position.bytes;
@ -186,7 +186,7 @@ static void ts_lexer_goto(Lexer *self, Length position) {
};
self->chunk = empty_chunk;
self->chunk_start = position.bytes;
self->chunk_size = 1;
self->chunk_size = 2;
}
self->token_start_position = position;