diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h index 105f3670..0e73bc67 100644 --- a/include/tree_sitter/parser.h +++ b/include/tree_sitter/parser.h @@ -108,16 +108,16 @@ struct TSLanguage { #define START_TOKEN() ts_lexer_start_token(lexer); -#define ADVANCE(state_index) \ - { \ - DEBUG_LEX("ADVANCE %d", state_index); \ - if (ts_lexer_is_done(lexer)) { \ - DEBUG_LEX("END"); \ - return NULL; \ - } \ - ts_lexer_advance(lexer); \ - lex_state = state_index; \ - goto next_state; \ +#define ADVANCE(state_index) \ + { \ + DEBUG_LEX("ADVANCE %d", state_index); \ + if (ts_lexer_is_done(lexer)) { \ + DEBUG_LEX("END"); \ + return ts_lexer_accept(lexer, ts_builtin_sym_error, 0); \ + } \ + ts_lexer_advance(lexer); \ + lex_state = state_index; \ + goto next_state; \ } #define ACCEPT_TOKEN(symbol) \ @@ -126,10 +126,10 @@ struct TSLanguage { return ts_lexer_accept(lexer, symbol, ts_hidden_symbol_flags[symbol]); \ } -#define LEX_ERROR() \ - { \ - DEBUG_LEX("ERROR"); \ - return NULL; \ +#define LEX_ERROR() \ + { \ + DEBUG_LEX("ERROR"); \ + return ts_lexer_accept(lexer, ts_builtin_sym_error, 0); \ } #define SHIFT(to_state_value) \ diff --git a/spec/runtime/document_spec.cc b/spec/runtime/document_spec.cc index 13c007b9..1feab8b7 100644 --- a/spec/runtime/document_spec.cc +++ b/spec/runtime/document_spec.cc @@ -134,8 +134,8 @@ describe("Document", [&]() { TSNode *last = ts_node_child(array, 2); AssertThat(ts_node_name(error), Equals("error")); - AssertThat(ts_node_pos(error), Equals(string(" [123,").length())) - AssertThat(ts_node_size(error), Equals(string(" @@@@@").length())) + AssertThat(ts_node_pos(error), Equals(string(" [123, ").length())) + AssertThat(ts_node_size(error), Equals(string("@@@@@").length())) AssertThat(ts_node_name(last), Equals("true")); AssertThat(ts_node_pos(last), Equals(string(" [123, @@@@@, ").length())) @@ -158,8 +158,8 @@ describe("Document", [&]() { TSNode *last = ts_node_child(array, 2); AssertThat(ts_node_name(error), Equals("error")); - AssertThat(ts_node_pos(error), Equals(string(" [123,").length())) - AssertThat(ts_node_size(error), Equals(string(" faaaaalse").length())) + AssertThat(ts_node_pos(error), Equals(string(" [123, ").length())) + AssertThat(ts_node_size(error), Equals(string("faaaaalse").length())) AssertThat(ts_node_name(last), Equals("true")); AssertThat(ts_node_pos(last), Equals(string(" [123, faaaaalse, ").length())) @@ -182,8 +182,8 @@ describe("Document", [&]() { TSNode *last = ts_node_child(array, 2); AssertThat(ts_node_name(error), Equals("error")); - AssertThat(ts_node_pos(error), Equals(string(" [123,").length())) - AssertThat(ts_node_size(error), Equals(string(" true false").length())) + AssertThat(ts_node_pos(error), Equals(string(" [123, ").length())) + AssertThat(ts_node_size(error), Equals(string("true false").length())) AssertThat(ts_node_name(last), Equals("true")); AssertThat(ts_node_pos(last), Equals(string(" [123, true false, ").length())) @@ -206,8 +206,8 @@ describe("Document", [&]() { TSNode *last = ts_node_child(array, 2); AssertThat(ts_node_name(error), Equals("error")); - AssertThat(ts_node_pos(error), Equals(string(" [123,").length())) - AssertThat(ts_node_size(error), Equals(string(" ").length())) + AssertThat(ts_node_pos(error), Equals(string(" [123, ").length())) + AssertThat(ts_node_size(error), Equals(0)) AssertThat(ts_node_name(last), Equals("true")); AssertThat(ts_node_pos(last), Equals(string(" [123, , ").length())) diff --git a/src/runtime/lexer.c b/src/runtime/lexer.c index 222517eb..d456ce49 100644 --- a/src/runtime/lexer.c +++ b/src/runtime/lexer.c @@ -26,7 +26,9 @@ static TSTree *accept(TSLexer *lexer, TSSymbol symbol, int is_hidden) { size_t size = current_position - lexer->token_start_position; size_t padding = lexer->token_start_position - lexer->token_end_position; lexer->token_end_position = current_position; - return ts_tree_make_leaf(symbol, size, padding, is_hidden); + return (symbol == ts_builtin_sym_error) + ? ts_tree_make_error(size, padding, ts_lexer_lookahead_char(lexer)) + : ts_tree_make_leaf(symbol, size, padding, is_hidden); } /* diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 1483cfbe..36ac5a40 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -121,9 +121,6 @@ static void lex(TSParser *parser, TSStateId lex_state) { if (parser->lookahead) ts_tree_release(parser->lookahead); parser->lookahead = parser->language->lex_fn(&parser->lexer, lex_state); - if (!parser->lookahead) { - parser->lookahead = build_error_node(parser); - } } static int handle_error(TSParser *parser) { @@ -137,7 +134,7 @@ static int handle_error(TSParser *parser) { * Unwind the parse stack until a state is found in which an error is * expected and the current lookahead token is expected afterwards. */ - size_t error_start_pos = last_token_end; + size_t error_start = last_token_end; TS_STACK_FROM_TOP(parser->stack, entry, i) { TSStateId state = entry->state; TSParseAction action_on_error = @@ -150,8 +147,7 @@ static int handle_error(TSParser *parser) { if (action_after_error.type != TSParseActionTypeError) { DEBUG_PARSE("RECOVER %u", state_after_error); - size_t current_position = ts_lexer_position(&parser->lexer); - error->size = current_position - 1 - error_start_pos; + error->size += ts_lexer_position(&parser->lexer) - 1 - error_start; ts_stack_shrink(&parser->stack, i + 1); ts_stack_push(&parser->stack, state_after_error, error); ts_tree_release(error); @@ -160,7 +156,7 @@ static int handle_error(TSParser *parser) { } TSTree *removed_tree = entry->node; - error_start_pos -= ts_tree_total_size(removed_tree); + error_start -= ts_tree_total_size(removed_tree); } /*