traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Automatically skip BOM characters at beginnings of files

Browse source 
Refs tree-sitter/tree-sitter-python#48
This commit is contained in:
Max Brunsfeld 2019-08-02 11:30:43 -07:00
parent 15629f1231
commit 0afbc31789
2 changed files with 65 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

6
lib/src/lexer.c
View file

@ -20,6 +20,8 @@
static const char empty_chunk[3] = { 0, 0 };
static const int32_t BYTE_ORDER_MARK = 0xFEFF;
static void ts_lexer__get_chunk(Lexer *self) {
self->chunk_start = self->current_position.bytes;
self->chunk = self->input.read(
@ -248,6 +250,10 @@ void ts_lexer_start(Lexer *self) {
self->data.result_symbol = 0;
if (!self->chunk) ts_lexer__get_chunk(self);
if (!self->lookahead_size) ts_lexer__get_lookahead(self);
if (
self->current_position.bytes == 0 &&
self->data.lookahead == BYTE_ORDER_MARK
) ts_lexer__advance((TSLexer *)self, true);
}
void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte) {