traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add TSInput option to measure columns in bytes not characters

Browse source
This commit is contained in:
Max Brunsfeld 2017-06-15 16:35:34 -07:00
parent b862db766e
commit c66fddd3aa
6 changed files with 26 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

9
src/runtime/document.c
View file

@ -26,8 +26,13 @@ void ts_document_free(TSDocument *self) {
parser_destroy(&self->parser);
if (self->tree)
ts_tree_release(self->tree);
ts_document_set_input(self,
(TSInput){ NULL, NULL, NULL, TSInputEncodingUTF8 });
ts_document_set_input(self, (TSInput){
NULL,
NULL,
NULL,
TSInputEncodingUTF8,
false
});
ts_free(self);
}

2
src/runtime/lexer.c
View file

@ -60,6 +60,8 @@ static void ts_lexer__advance(void *payload, bool skip) {
if (self->data.lookahead == '\n') {
self->current_position.extent.row++;
self->current_position.extent.column = 0;
} else if (self->input.measure_columns_in_bytes) {
self->current_position.extent.column += self->lookahead_size;
} else {
self->current_position.extent.column++;
}

3
src/runtime/string_input.c
View file

@ -43,8 +43,9 @@ TSInput ts_string_input_make_with_length(const char *string, uint32_t length) {
.read = ts_string_input_read,
.seek = ts_string_input_seek,
.encoding = TSInputEncodingUTF8,
.measure_columns_in_bytes = false,
};
error:
return (TSInput){ NULL, NULL, NULL, TSInputEncodingUTF8 };
return (TSInput){ NULL, NULL, NULL, TSInputEncodingUTF8, false };
}