Convert repeat rules into pairs of recursive rules

src/compiler/build_tables/item.cpp

@@ -40,6 +40,7 @@ namespace tree_sitter {
         
         bool LexItem::operator<(const LexItem &other) const {
             if (rule_name < other.rule_name) return true;
+            if (rule_name > other.rule_name) return false;
             if (rule->to_string() < other.rule->to_string()) return true;
             return false;
         }
@@ -50,6 +51,7 @@ namespace tree_sitter {
             if (rule->to_string() < other.rule->to_string()) return true;
             if (rule->to_string() > other.rule->to_string()) return false;
             if (consumed_sym_count < other.consumed_sym_count) return true;
+            if (consumed_sym_count > other.consumed_sym_count) return false;
             if (lookahead_sym_name < other.lookahead_sym_name) return true;
             return false;
         }

src/compiler/prepare_grammar/expand_repeats.cpp

@@ -0,0 +1,58 @@
+#include "expand_repeats.h"
+#include <unordered_map>
+
+using std::string;
+using std::to_string;
+using std::unordered_map;
+using namespace tree_sitter::rules;
+
+namespace tree_sitter {
+    namespace prepare_grammar {
+        class RepeatExpander : rules::Visitor {
+        public:
+            rule_ptr value;
+            unordered_map<string, const rule_ptr> aux_rules;
+            
+            rule_ptr apply(const rule_ptr rule) {
+                rule->accept(*this);
+                return value;
+            }
+            
+            rule_ptr make_repeat_helper(string name, const rule_ptr &rule) {
+                return seq({
+                    rule,
+                    choice({ sym(name), blank() })
+                });
+            }
+            
+            void visit(const Repeat *rule) {
+                rule_ptr inner_rule = apply(rule->content);
+                string helper_rule_name = string("repeat_helper") + to_string(aux_rules.size() + 1);
+                aux_rules.insert({ helper_rule_name, make_repeat_helper(helper_rule_name, inner_rule) });
+                value = sym(helper_rule_name);
+            }
+            
+            void visit(const Seq *rule) {
+                value = seq({ apply(rule->left), apply(rule->right) });
+            }
+
+            void visit(const Choice *rule) {
+                value = choice({ apply(rule->left), apply(rule->right) });
+            }
+
+            void default_visit(const Rule *rule) {
+                value = rule->copy();
+            }
+        };
+        
+        Grammar expand_repeats(const Grammar &grammar) {
+            unordered_map<string, const rule_ptr> result;
+            RepeatExpander visitor;
+            for (auto pair : grammar.rules)
+                result.insert({ pair.first, visitor.apply(pair.second) });
+            for (auto pair : visitor.aux_rules)
+                result.insert(pair);
+            return Grammar(grammar.start_rule_name, result);
+        }
+    }
+}

src/compiler/prepare_grammar/expand_repeats.h

@@ -0,0 +1,12 @@
+#ifndef __tree_sitter__expand_repeats__
+#define __tree_sitter__expand_repeats__
+
+#include "grammar.h"
+
+namespace tree_sitter {
+    namespace prepare_grammar {
+        Grammar expand_repeats(const Grammar &);
+    }
+}
+
+#endif

src/compiler/prepare_grammar/perform.cpp

@@ -1,12 +1,14 @@
 #include "./perform.h"
-#include "extract_tokens.h"
+#include "./extract_tokens.h"
+#include "./expand_repeats.h"
 
 using std::pair;
 
 namespace tree_sitter {
     namespace prepare_grammar {
         pair<Grammar, Grammar> perform(const Grammar &input_grammar) {
-            return prepare_grammar::extract_tokens(input_grammar);
+            auto rule_grammar = expand_repeats(input_grammar);
+            return prepare_grammar::extract_tokens(rule_grammar);
         }
     }
 }