In lex error state, don't look for tokens that would match *any* line

src/compiler/build_tables/build_lex_table.cc

@@ -9,6 +9,7 @@
 #include "compiler/build_tables/get_completion_status.h"
 #include "compiler/build_tables/get_metadata.h"
 #include "compiler/build_tables/lex_item.h"
+#include "compiler/build_tables/does_match_any_line.h"
 #include "compiler/parse_table.h"
 #include "compiler/lexical_grammar.h"
 #include "compiler/rules/built_in_symbols.h"
@@ -50,18 +51,18 @@ class LexTableBuilder {
 
   LexTable build() {
     for (ParseState &parse_state : parse_table->states) {
-      LexItemSet item_set = build_lex_item_set(parse_state.expected_inputs());
+      LexItemSet item_set = build_lex_item_set(parse_state.expected_inputs(), false);
       parse_state.lex_state_id = add_lex_state(item_set);
     }
 
-    LexItemSet error_item_set = build_lex_item_set(parse_table->symbols);
+    LexItemSet error_item_set = build_lex_item_set(parse_table->symbols, true);
     populate_lex_state(error_item_set, LexTable::ERROR_STATE_ID);
 
     return lex_table;
   }
 
  private:
-  LexItemSet build_lex_item_set(const set<Symbol> &symbols) {
+  LexItemSet build_lex_item_set(const set<Symbol> &symbols, bool error) {
     LexItemSet result;
     for (const Symbol &symbol : symbols) {
       vector<rule_ptr> rules;
@@ -71,6 +72,9 @@ class LexTableBuilder {
         rules.push_back(CharacterSet().include(0).copy());
       } else if (symbol.is_token) {
         rule_ptr rule = lex_grammar.variables[symbol.index].rule;
+        if (error && does_match_any_line(rule))
+          continue;
+
         auto choice = rule->as<rules::Choice>();
         if (choice)
           for (const rule_ptr &element : choice->elements)

src/compiler/build_tables/does_match_any_line.cc

@@ -0,0 +1,62 @@
+#include "compiler/build_tables/does_match_any_line.h"
+#include "compiler/rules/choice.h"
+#include "compiler/rules/character_set.h"
+#include "compiler/rules/repeat.h"
+#include "compiler/rules/visitor.h"
+#include "compiler/rules/seq.h"
+#include "compiler/rules/metadata.h"
+
+namespace tree_sitter {
+namespace build_tables {
+
+class DoesTokenCatchAnyCharacter : public rules::RuleFn<bool> {
+  bool apply_to(const rules::Choice *rule) {
+    for (const rule_ptr &element : rule->elements)
+      if (apply(element))
+        return true;
+    return false;
+  }
+
+  bool apply_to(const rules::Metadata *rule) {
+    return apply(rule->rule);
+  }
+
+  bool apply_to(const rules::CharacterSet *rule) {
+    if (rule->includes_all) {
+      for (uint32_t character : rule->excluded_chars) {
+        if (character != 0 && character != '\n')
+          return false;
+      }
+      return true;
+    }
+    return false;
+  }
+};
+
+class DoesTokenCatchAll : public rules::RuleFn<bool> {
+  bool apply_to(const rules::Repeat *rule) {
+    return DoesTokenCatchAnyCharacter().apply(rule->content);
+  }
+
+  bool apply_to(const rules::Metadata *rule) {
+    return apply(rule->rule);
+  }
+
+  bool apply_to(const rules::Choice *rule) {
+    for (const rule_ptr &element : rule->elements)
+      if (apply(element))
+        return true;
+    return false;
+  }
+
+  bool apply_to(const rules::Seq *rule) {
+    return apply(rule->left) && apply(rule->right);
+  }
+};
+
+bool does_match_any_line(const rule_ptr &rule) {
+  return DoesTokenCatchAll().apply(rule);
+}
+
+}  // namespace build_tables
+}  // namespace tree_sitter

src/compiler/build_tables/does_match_any_line.h

@@ -0,0 +1,14 @@
+#ifndef COMPILER_BUILD_TABLES_DOES_MATCH_ANY_LINE_H_
+#define COMPILER_BUILD_TABLES_DOES_MATCH_ANY_LINE_H_
+
+#include "tree_sitter/compiler.h"
+
+namespace tree_sitter {
+namespace build_tables {
+
+bool does_match_any_line(const rule_ptr &);
+
+}  // namespace build_tables
+}  // namespace tree_sitter
+
+#endif  // COMPILER_BUILD_TABLES_DOES_MATCH_ANY_LINE_H_