diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h index 603015ac..5b4dcb09 100644 --- a/include/tree_sitter/parser.h +++ b/include/tree_sitter/parser.h @@ -95,7 +95,7 @@ struct TSLanguage { const TSParseActionEntry *parse_actions; const TSStateId *lex_states; const TSParseAction *recovery_actions; - void (*lex_fn)(TSLexer *, TSStateId, bool); + bool (*lex_fn)(TSLexer *, TSStateId, bool); }; /* @@ -129,23 +129,23 @@ struct TSLanguage { { \ lexer->result_is_fragile = true; \ lexer->result_symbol = symbol_value; \ - return; \ + return true; \ } #define ACCEPT_TOKEN(symbol_value) \ { \ lexer->result_symbol = symbol_value; \ - return; \ + return true; \ } #define LEX_ERROR() \ if (error_mode) { \ if (state == ts_lex_state_error) \ lexer->advance(lexer, state, TSTransitionTypeError); \ + GO_TO_STATE(ts_lex_state_error); \ } else { \ - error_mode = true; \ - } \ - GO_TO_STATE(ts_lex_state_error) + return false; \ + } /* * Parse Table Macros diff --git a/spec/fixtures/error_corpus/javascript_errors.txt b/spec/fixtures/error_corpus/javascript_errors.txt index ca2a125e..0121f18d 100644 --- a/spec/fixtures/error_corpus/javascript_errors.txt +++ b/spec/fixtures/error_corpus/javascript_errors.txt @@ -14,8 +14,8 @@ if (x) { (program (expression_statement - (ERROR (identifier) (identifier)) - (identifier)) + (identifier) + (ERROR (identifier) (identifier))) (expression_statement (function_call (identifier))) (if_statement (identifier) (statement_block (ERROR) @@ -43,12 +43,12 @@ Invalid for loops ok1; -for (aa bb cc; dd; ee) +for (a b c; d; e) wat(); ok2; -for (a; b; c d ef) +for (a; b; c d e) wat(); --- diff --git a/src/compiler/generate_code/c_code.cc b/src/compiler/generate_code/c_code.cc index 4ab67644..2a0a528f 100644 --- a/src/compiler/generate_code/c_code.cc +++ b/src/compiler/generate_code/c_code.cc @@ -192,8 +192,7 @@ class CCodeGenerator { void add_lex_function() { line( - "static void ts_lex(TSLexer *lexer, TSStateId state, bool error_mode) " - "{"); + "static bool ts_lex(TSLexer *lexer, TSStateId state, bool error_mode) {"); indent([&]() { line("START_LEXER();"); _switch("state", [&]() { diff --git a/src/runtime/parser.c b/src/runtime/parser.c index ab257889..77b6305b 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -233,9 +233,16 @@ static bool ts_parser__can_reuse(TSParser *self, StackVersion version, } static TSTree *ts_parser__lex(TSParser *self, TSStateId state, bool error_mode) { - TSLexerResult lex_result; + TSLength position = self->lexer.current_position; + ts_lexer_start(&self->lexer, state); - self->language->lex_fn(&self->lexer, state, error_mode); + if (!self->language->lex_fn(&self->lexer, state, error_mode)) { + ts_lexer_reset(&self->lexer, position); + ts_lexer_start(&self->lexer, state); + assert(self->language->lex_fn(&self->lexer, 0, true)); + } + + TSLexerResult lex_result; ts_lexer_finish(&self->lexer, &lex_result); TSTree *result;