Make token extraction work for repeat rules

spec/compiler/prepare_grammar/extract_tokens_spec.cpp

@@ -54,6 +54,30 @@ describe("preparing a grammar", []() {
                 character('b') }) },
         })));
     });
+    
+    it("looks inside sequences, choices and repeats", [&]() {
+        auto result = extract_tokens(Grammar({
+            { "rule1", seq({
+                choice({
+                    repeat(choice({ str("stuff"), sym("a") })),
+                    sym("b"),
+                }),
+                sym("c") }) }
+        }));
+        
+        AssertThat(result.first, Equals(Grammar({
+            { "rule1", seq({
+                choice({
+                    repeat(choice({ sym("1"), sym("a") })),
+                    sym("b"),
+                }),
+                sym("c") }) }
+        })));
+        
+        AssertThat(result.second, Equals(Grammar("", {
+            { "1", str("stuff") },
+        })));
+    });
 });
 
 END_TEST

src/compiler/prepare_grammar/extract_tokens.cpp

@@ -53,6 +53,10 @@ namespace tree_sitter {
             void visit(const Seq *rule) {
                 value = seq({ apply(rule->left), apply(rule->right) });
             }
+            
+            void visit(const Repeat *rule) {
+                value = repeat(apply(rule->content));
+            }
         };
         
         pair<Grammar, Grammar> extract_tokens(const Grammar &input_grammar) {

src/compiler/prepare_grammar/search_for_symbols.cpp

@@ -26,6 +26,10 @@ namespace tree_sitter {
             void visit(const rules::Seq *seq) {
                 value = apply(seq->left) || apply(seq->right);
             }
+            
+            void visit(const rules::Repeat *rule) {
+                value = apply(rule->content);
+            }
         };
         
         bool search_for_symbols(const rules::rule_ptr &rule) {