Normalize lexical grammar rules before constructing lex table

src/compiler/prepare_grammar/normalize_rules.cc

@@ -0,0 +1,19 @@
+#include "compiler/prepare_grammar/normalize_rules.h"
+#include "compiler/prepare_grammar/extract_choices.h"
+#include "compiler/rules/choice.h"
+
+namespace tree_sitter {
+namespace prepare_grammar {
+
+LexicalGrammar normalize_rules(const LexicalGrammar &input_grammar) {
+  LexicalGrammar result(input_grammar);
+
+  for (Variable &variable : result.variables) {
+    variable.rule = rules::Choice::build(extract_choices(variable.rule));
+  }
+
+  return result;
+}
+
+}  // namespace prepare_grammar
+}  // namespace tree_sitter