diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h index d5eca277..ea0d6d2d 100644 --- a/include/tree_sitter/parser.h +++ b/include/tree_sitter/parser.h @@ -39,6 +39,7 @@ typedef struct { typedef struct { ts_input input; const char *current_chunk; + size_t current_chunk_end; size_t position; size_t token_end_position; size_t token_start_position; @@ -53,12 +54,15 @@ static void ts_lex(ts_parser *parser); static const ts_symbol * ts_recover(ts_state state, ts_state *to_state, size_t *count); static ts_parser ts_parser_make(ts_input input) { + size_t bytes_read = 0; + const char *chunk = input.read_fn(input.data, &bytes_read); ts_parser result = { .input = input, .token_start_position = 0, .token_end_position = 0, .position = 0, - .current_chunk = input.read_fn(input.data), + .current_chunk = chunk, + .current_chunk_end = bytes_read, .lookahead_node = NULL, .prev_lookahead_node = NULL, .lex_state = 0, @@ -140,11 +144,12 @@ static void ts_parser_reduce(ts_parser *parser, ts_symbol symbol, int immediate_ } static void ts_parser_advance(ts_parser *parser) { - if (parser->current_chunk && parser->current_chunk[parser->position]) { + if (parser->position < parser->current_chunk_end) { parser->position++; } else { - parser->current_chunk = parser->input.read_fn(parser->input.data); - parser->position = 0; + size_t bytes_read = 0; + parser->current_chunk = parser->input.read_fn(parser->input.data, &bytes_read); + parser->current_chunk_end += bytes_read; } } diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index 604b98a4..6f3a4bca 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -4,13 +4,13 @@ #ifdef __cplusplus extern "C" { #endif - + #include typedef int ts_symbol; static const ts_symbol ts_builtin_sym_error = -1; static const ts_symbol ts_builtin_sym_end = -2; - + typedef struct ts_tree { ts_symbol symbol; size_t ref_count; @@ -42,7 +42,7 @@ ts_tree ** ts_tree_children(const ts_tree *tree); typedef struct { void *data; - const char * (* read_fn)(void *data); + const char * (* read_fn)(void *data, size_t *bytes_read); int (* seek_fn)(void *data, size_t position); void (* release_fn)(void *data); } ts_input; @@ -61,7 +61,7 @@ void ts_document_set_input(ts_document *, ts_input input); void ts_document_set_input_string(ts_document *, const char *text); const ts_tree * ts_document_tree(const ts_document *); const char * ts_document_string(const ts_document *); - + #ifdef __cplusplus } #endif diff --git a/src/runtime/document.c b/src/runtime/document.c index 9cce0c58..b5b31bdd 100644 --- a/src/runtime/document.c +++ b/src/runtime/document.c @@ -37,8 +37,9 @@ typedef struct { size_t length; } ts_string_input_data; -const char * ts_string_input_read(void *d) { +const char * ts_string_input_read(void *d, size_t *bytes_read) { ts_string_input_data *data = (ts_string_input_data *)d; + *bytes_read = data->length; return data->string + data->position; }