Rewind and re-tokenize in error mode after detecting an error

include/tree_sitter/parser.h

@@ -95,7 +95,7 @@ struct TSLanguage {
   const TSParseActionEntry *parse_actions;
   const TSStateId *lex_states;
   const TSParseAction *recovery_actions;
-  void (*lex_fn)(TSLexer *, TSStateId, bool);
+  bool (*lex_fn)(TSLexer *, TSStateId, bool);
 };
 
 /*
@@ -129,23 +129,23 @@ struct TSLanguage {
   {                                        \
     lexer->result_is_fragile = true;       \
     lexer->result_symbol = symbol_value;   \
-    return;                                \
+    return true;                           \
   }
 
 #define ACCEPT_TOKEN(symbol_value)       \
   {                                      \
     lexer->result_symbol = symbol_value; \
-    return;                              \
+    return true;                         \
   }
 
 #define LEX_ERROR()                                        \
   if (error_mode) {                                        \
     if (state == ts_lex_state_error)                       \
       lexer->advance(lexer, state, TSTransitionTypeError); \
+    GO_TO_STATE(ts_lex_state_error);                       \
   } else {                                                 \
-    error_mode = true;                                     \
-  }                                                        \
-  GO_TO_STATE(ts_lex_state_error)
+    return false;                                          \
+  }
 
 /*
  *  Parse Table Macros

spec/fixtures/error_corpus/javascript_errors.txt

@@ -14,8 +14,8 @@ if (x) {
 
 (program
   (expression_statement
-    (ERROR (identifier) (identifier))
-    (identifier))
+    (identifier)
+    (ERROR (identifier) (identifier)))
   (expression_statement (function_call (identifier)))
   (if_statement (identifier) (statement_block
     (ERROR)
@@ -43,12 +43,12 @@ Invalid for loops
 
 ok1;
 
-for (aa bb cc; dd; ee)
+for (a b c; d; e)
   wat();
 
 ok2;
 
-for (a; b; c d ef)
+for (a; b; c d e)
   wat();
 
 ---

src/compiler/generate_code/c_code.cc

@@ -192,8 +192,7 @@ class CCodeGenerator {
 
   void add_lex_function() {
     line(
-      "static void ts_lex(TSLexer *lexer, TSStateId state, bool error_mode) "
-      "{");
+      "static bool ts_lex(TSLexer *lexer, TSStateId state, bool error_mode) {");
     indent([&]() {
       line("START_LEXER();");
       _switch("state", [&]() {

src/runtime/parser.c

@@ -233,9 +233,16 @@ static bool ts_parser__can_reuse(TSParser *self, StackVersion version,
 }
 
 static TSTree *ts_parser__lex(TSParser *self, TSStateId state, bool error_mode) {
-  TSLexerResult lex_result;
+  TSLength position = self->lexer.current_position;
+
   ts_lexer_start(&self->lexer, state);
-  self->language->lex_fn(&self->lexer, state, error_mode);
+  if (!self->language->lex_fn(&self->lexer, state, error_mode)) {
+    ts_lexer_reset(&self->lexer, position);
+    ts_lexer_start(&self->lexer, state);
+    assert(self->language->lex_fn(&self->lexer, 0, true));
+  }
+
+  TSLexerResult lex_result;
   ts_lexer_finish(&self->lexer, &lex_result);
 
   TSTree *result;