From af7f57a80e23b8c9d34a631c4900a010789fe91f Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Sun, 5 Oct 2014 16:56:50 -0700 Subject: [PATCH] Fix sizing of error nodes after edits --- spec/runtime/parser_spec.cc | 32 ++++++++++++++++++++++++++++++++ src/runtime/lexer.c | 29 +++++++++++++++++++---------- src/runtime/lexer.h | 2 +- src/runtime/parser.c | 22 ++++++++++++++-------- 4 files changed, 66 insertions(+), 19 deletions(-) diff --git a/spec/runtime/parser_spec.cc b/spec/runtime/parser_spec.cc index 064f8465..0f652efb 100644 --- a/spec/runtime/parser_spec.cc +++ b/spec/runtime/parser_spec.cc @@ -29,19 +29,28 @@ describe("Parser", [&]() { reader = new SpyReader(text, chunk_size); ts_document_set_input(doc, reader->input()); root = ts_document_root_node(doc); + AssertThat(ts_node_size(root).bytes + ts_node_pos(root).bytes, Equals(strlen(text))); reader->clear(); }; auto insert_text = [&](size_t position, string text) { + size_t prev_size = ts_node_size(root).bytes + ts_node_pos(root).bytes; AssertThat(reader->insert(position, text), IsTrue()); ts_document_edit(doc, { position, 0, text.length() }); + root = ts_document_root_node(doc); + size_t new_size = ts_node_size(root).bytes + ts_node_pos(root).bytes; + AssertThat(new_size, Equals(prev_size + text.size())); }; auto delete_text = [&](size_t position, size_t length) { + size_t prev_size = ts_node_size(root).bytes + ts_node_pos(root).bytes; AssertThat(reader->erase(position, length), IsTrue()); ts_document_edit(doc, { position, length, 0 }); + root = ts_document_root_node(doc); + size_t new_size = ts_node_size(root).bytes + ts_node_pos(root).bytes; + AssertThat(new_size, Equals(prev_size - length)); }; describe("handling errors", [&]() { @@ -223,6 +232,29 @@ describe("Parser", [&]() { }); }); + describe("introducing an error", [&]() { + it("gives the error the right size", [&]() { + ts_document_set_language(doc, ts_language_javascript()); + + set_text("var x = y;"); + + AssertThat(ts_node_string(root), Equals( + "(DOCUMENT (var_declaration " + "(identifier) (identifier)))")); + + insert_text(strlen("var x = y"), " *"); + + AssertThat(ts_node_string(root), Equals( + "(DOCUMENT (var_declaration (ERROR ';')))")); + + insert_text(strlen("var x = y *"), " z"); + + AssertThat(ts_node_string(root), Equals( + "(DOCUMENT (var_declaration " + "(identifier) (math_op (identifier) (identifier))))")); + }); + }); + describe("new tokens near the beginning of the input", [&]() { before_each([&]() { set_text("123 * 456"); diff --git a/src/runtime/lexer.c b/src/runtime/lexer.c index f7c28a2c..898d6f75 100644 --- a/src/runtime/lexer.c +++ b/src/runtime/lexer.c @@ -61,19 +61,28 @@ static TSTree *accept(TSLexer *lexer, TSSymbol symbol, int is_hidden) { * this library. */ TSLexer ts_lexer_make() { - TSLexer result = - (TSLexer) { .debug = 0, .advance_fn = advance, .accept_fn = accept, }; - ts_lexer_reset(&result); + TSLexer result = (TSLexer) { .advance_fn = advance, + .accept_fn = accept, + .debug = 0, + .chunk = NULL, + .chunk_start = 0, + .chunk_size = 0, + .current_position = ts_length_zero(), + .token_start_position = ts_length_zero(), + .token_end_position = ts_length_zero(), + .lookahead = 0, + .lookahead_size = 0, }; return result; } -void ts_lexer_reset(TSLexer *lexer) { - lexer->chunk = NULL; - lexer->chunk_start = 0; - lexer->chunk_size = 0; - lexer->current_position = ts_length_zero(); - lexer->token_start_position = ts_length_zero(); - lexer->token_end_position = ts_length_zero(); +void ts_lexer_reset(TSLexer *lexer, TSLength position) { + lexer->input.seek_fn(lexer->input.data, position); + lexer->current_position = position; + lexer->token_end_position = position; lexer->lookahead = 0; lexer->lookahead_size = 0; + lexer->chunk_start = position.bytes; + lexer->chunk_size = 0; + lexer->chunk = lexer->input.read_fn(lexer->input.data, &lexer->chunk_size); + ts_lexer_advance(lexer); } diff --git a/src/runtime/lexer.h b/src/runtime/lexer.h index a557872b..b127d483 100644 --- a/src/runtime/lexer.h +++ b/src/runtime/lexer.h @@ -8,7 +8,7 @@ extern "C" { #include "tree_sitter/parser.h" TSLexer ts_lexer_make(); -void ts_lexer_reset(TSLexer *); +void ts_lexer_reset(TSLexer *, TSLength); #ifdef __cplusplus } diff --git a/src/runtime/parser.c b/src/runtime/parser.c index b00365ad..973c0e02 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -124,6 +124,13 @@ static void lex(TSParser *parser, TSStateId lex_state) { parser->lookahead = parser->language->lex_fn(&parser->lexer, lex_state); } +static void resize_error(TSParser *parser, TSTree *error) { + error->size = + ts_length_sub(ts_length_sub(parser->lexer.token_start_position, + ts_stack_right_position(&parser->stack)), + error->padding); +} + static int handle_error(TSParser *parser) { TSTree *error = parser->lookahead; ts_tree_retain(error); @@ -149,10 +156,8 @@ static int handle_error(TSParser *parser) { ts_stack_shrink(&parser->stack, entry - parser->stack.entries + 1); parser->lookahead->padding = ts_length_zero(); - error->size = ts_length_sub( - ts_length_sub(parser->lexer.token_start_position, - ts_stack_right_position(&parser->stack)), - error->padding); + + resize_error(parser, error); ts_stack_push(&parser->stack, state_after_error, error); ts_tree_release(error); return 1; @@ -176,6 +181,8 @@ static int handle_error(TSParser *parser) { if (ts_length_eq(parser->lexer.current_position, prev_position)) if (!ts_lexer_advance(&parser->lexer)) { DEBUG_PARSE("FAIL TO RECOVER"); + + resize_error(parser, error); ts_stack_push(&parser->stack, 0, error); ts_tree_release(error); return 0; @@ -220,11 +227,10 @@ const TSTree *ts_parser_parse(TSParser *parser, TSInput input, TSInputEdit *edit) { parser->lookahead = NULL; parser->next_lookahead = NULL; - ts_lexer_reset(&parser->lexer); - parser->lexer.input = input; + TSLength position = breakdown_stack(parser, edit); - input.seek_fn(input.data, breakdown_stack(parser, edit)); - ts_lexer_advance(&parser->lexer); + parser->lexer.input = input; + ts_lexer_reset(&parser->lexer, position); for (;;) { TSParseAction action = next_action(parser);