Perform keyword optimization using explicitly selected word token

rather than trying to infer the word token automatically.

Co-Authored-By: Ashi Krishnan <queerviolet@github.com>

src/compiler/prepare_grammar/expand_repeats.cc

@@ -106,6 +106,7 @@ InitialSyntaxGrammar expand_repeats(const InitialSyntaxGrammar &grammar) {
     expander.aux_rules.end()
   );
 
+  result.word_rule = grammar.word_rule;
   return result;
 }
 

src/compiler/prepare_grammar/extract_tokens.cc

@@ -329,6 +329,18 @@ tuple<InitialSyntaxGrammar, LexicalGrammar, CompileError> extract_tokens(
     }
   }
 
+  syntax_grammar.word_rule = symbol_replacer.replace_symbol(grammar.word_rule);
+  if (syntax_grammar.word_rule.is_non_terminal()) {
+    return make_tuple(
+      syntax_grammar,
+      lexical_grammar,
+      CompileError(
+        TSCompileErrorTypeInvalidWordRule,
+        "Word rules must be tokens"
+      )
+    );
+  }
+
   return make_tuple(syntax_grammar, lexical_grammar, CompileError::none());
 }
 

src/compiler/prepare_grammar/flatten_grammar.cc

@@ -161,6 +161,8 @@ pair<SyntaxGrammar, CompileError> flatten_grammar(const InitialSyntaxGrammar &gr
     i++;
   }
 
+  result.word_rule = grammar.word_rule;
+  
   return {result, CompileError::none()};
 }
 

src/compiler/prepare_grammar/initial_syntax_grammar.h

@@ -17,6 +17,7 @@ struct InitialSyntaxGrammar {
   std::set<std::set<rules::Symbol>> expected_conflicts;
   std::vector<ExternalToken> external_tokens;
   std::set<rules::Symbol> variables_to_inline;
+  rules::Symbol word_rule = rules::NONE();
 };
 
 }  // namespace prepare_grammar

src/compiler/prepare_grammar/intern_symbols.cc

@@ -166,6 +166,8 @@ pair<InternedGrammar, CompileError> intern_symbols(const InputGrammar &grammar)
     }
   }
 
+  result.word_rule = interner.intern_symbol(grammar.word_rule);
+
   return {result, CompileError::none()};
 }
 

src/compiler/prepare_grammar/interned_grammar.h

@@ -15,8 +15,8 @@ struct InternedGrammar {
   std::vector<rules::Rule> extra_tokens;
   std::set<std::set<rules::Symbol>> expected_conflicts;
   std::vector<Variable> external_tokens;
-  std::set<rules::Symbol> blank_external_tokens;
   std::set<rules::Symbol> variables_to_inline;
+  rules::Symbol word_rule;
 };
 
 }  // namespace prepare_grammar