traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/check-utf8proc_iterate-return' into update-fixture-grammars

Browse source
This commit is contained in:
Max Brunsfeld 2017-03-21 09:59:35 -07:00
commit 63fb041961
2 changed files with 20 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

14
src/runtime/lexer.c
View file

@ -36,11 +36,17 @@ static void ts_lexer__get_lookahead(Lexer *self) {
const uint8_t *chunk = (const uint8_t *)self->chunk + position_in_chunk;
uint32_t size = self->chunk_size - position_in_chunk + 1;
if (self->input.encoding == TSInputEncodingUTF8)
self->lookahead_size =
utf8proc_iterate(chunk, size, &self->data.lookahead);
else
if (self->input.encoding == TSInputEncodingUTF8) {
int64_t lookahead_size = utf8proc_iterate(chunk, size, &self->data.lookahead);
if (lookahead_size < 0) {
self->lookahead_size = 1;
} else {
self->lookahead_size = lookahead_size;
}
}
else {
self->lookahead_size = utf16_iterate(chunk, size, &self->data.lookahead);
}
}
static void ts_lexer__advance(void *payload, bool skip) {