Don't let lex state merging be fooled by trivial loops

src/build_tables/build_lex_table.rs

@@ -168,7 +168,7 @@ impl<'a> LexTableBuilder<'a> {
             self.table.states[state_id].advance_actions.push((
                 CharacterSet::empty().add_char('\0'),
                 AdvanceAction {
-                    state: next_state_id,
+                    state: Some(next_state_id),
                     in_main_token: true,
                 },
             ));
@@ -189,10 +189,15 @@ impl<'a> LexTableBuilder<'a> {
                 }
             }
             let (next_state_id, _) = self.add_state(states, eof_valid && is_separator);
+            let next_state = if next_state_id == state_id {
+                None
+            } else {
+                Some(next_state_id)
+            };
             self.table.states[state_id].advance_actions.push((
                 characters,
                 AdvanceAction {
-                    state: next_state_id,
+                    state: next_state,
                     in_main_token: !is_separator,
                 },
             ));
@@ -231,10 +236,10 @@ fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) {
             }
         }
         for state in table.states.iter_mut() {
-            for advance_action in state.advance_actions.iter_mut() {
-                if let Some(new_state_id) = state_replacements.get(&advance_action.1.state) {
-                    advance_action.1.state = *new_state_id;
-                }
+            for (_, advance_action) in state.advance_actions.iter_mut() {
+                advance_action.state = advance_action
+                    .state
+                    .map(|s| state_replacements.get(&s).cloned().unwrap_or(s))
             }
         }
     }
@@ -259,8 +264,8 @@ fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) {
     }
 
     for state in table.states.iter_mut() {
-        for advance_action in state.advance_actions.iter_mut() {
-            advance_action.1.state = final_state_replacements[advance_action.1.state];
+        for (_, advance_action) in state.advance_actions.iter_mut() {
+            advance_action.state = advance_action.state.map(|s| final_state_replacements[s]);
         }
     }
 

src/render/mod.rs

@@ -342,7 +342,7 @@ impl Generator {
         for (i, state) in lex_table.states.into_iter().enumerate() {
             add_line!(self, "case {}:", i);
             indent!(self);
-            self.add_lex_state(state);
+            self.add_lex_state(i, state);
             dedent!(self);
         }
 
@@ -358,7 +358,7 @@ impl Generator {
         add_line!(self, "");
     }
 
-    fn add_lex_state(&mut self, state: LexState) {
+    fn add_lex_state(&mut self, index: usize, state: LexState) {
         if let Some(accept_action) = state.accept_action {
             add_line!(self, "ACCEPT_TOKEN({});", self.symbol_ids[&accept_action]);
         }
@@ -372,14 +372,14 @@ impl Generator {
             if self.add_character_set_condition(&characters, &ruled_out_characters) {
                 add!(self, ")\n");
                 indent!(self);
-                self.add_advance_action(&action);
+                self.add_advance_action(index, &action);
                 if let CharacterSet::Include(chars) = characters {
                     ruled_out_characters.extend(chars.iter().map(|c| *c as u32));
                 }
                 dedent!(self);
             } else {
                 self.buffer.truncate(previous_length);
-                self.add_advance_action(&action);
+                self.add_advance_action(index, &action);
             }
         }
 
@@ -491,11 +491,11 @@ impl Generator {
             })
     }
 
-    fn add_advance_action(&mut self, action: &AdvanceAction) {
+    fn add_advance_action(&mut self, index: usize, action: &AdvanceAction) {
         if action.in_main_token {
-            add_line!(self, "ADVANCE({});", action.state);
+            add_line!(self, "ADVANCE({});", action.state.unwrap_or(index));
         } else {
-            add_line!(self, "SKIP({});", action.state);
+            add_line!(self, "SKIP({});", action.state.unwrap_or(index));
         }
     }
 

src/tables.rs

@@ -48,7 +48,7 @@ pub(crate) struct ParseTable {
 
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub(crate) struct AdvanceAction {
-    pub state: LexStateId,
+    pub state: Option<LexStateId>,
     pub in_main_token: bool,
 }