Refactor logic for marking '_'-prefixed rules as hidden

spec/compiler/helpers/containers.h

@@ -3,14 +3,12 @@
 
 #include <map>
 #include <vector>
-#include <string>
 #include <initializer_list>
 #include "tree_sitter/compiler.h"
 #include "compiler/rule.h"
 
 using std::map;
 using std::vector;
-using std::string;
 using std::initializer_list;
 using std::pair;
 using tree_sitter::rule_ptr;
@@ -31,23 +29,6 @@ class rule_map : public map<K, rule_ptr> {
   rule_map(const initializer_list<pair<const K, rule_ptr>> &list) : map<K, rule_ptr>(list) {}
 };
 
-class rule_list : public vector<pair<string, rule_ptr>> {
- public:
-  bool operator==(const vector<pair<string, rule_ptr>> &other) const {
-    if (this->size() != other.size()) return false;
-    for (size_t i = 0; i < this->size(); i++) {
-      auto pair = this->operator[](i);
-      auto other_pair = other[i];
-      if (!pair.second->operator==(*other_pair.second))
-        return false;
-    }
-    return true;
-  }
-
-  rule_list(const initializer_list<pair<string, rule_ptr>> &list) :
-      vector<pair<string, rule_ptr>>(list) {}
-};
-
 template<typename T>
 class eq_vector : public vector<T> {
  public:

spec/compiler/prepare_grammar/expand_repeats_spec.cc

@@ -32,7 +32,7 @@ describe("expand_repeats", []() {
           i_token(0),
           choice({ i_sym(1), blank() })
         }),
-        RuleEntryTypeHidden
+        RuleEntryTypeAuxiliary
       },
     })));
   });
@@ -66,7 +66,7 @@ describe("expand_repeats", []() {
           i_token(11),
           choice({ i_sym(1), blank() })
         }),
-        RuleEntryTypeHidden
+        RuleEntryTypeAuxiliary
       },
     })));
   });
@@ -94,7 +94,7 @@ describe("expand_repeats", []() {
           i_token(11),
           choice({ i_sym(1), blank() }),
         }),
-        RuleEntryTypeHidden
+        RuleEntryTypeAuxiliary
       },
     })));
   });
@@ -138,7 +138,7 @@ describe("expand_repeats", []() {
           i_token(4),
           choice({ i_sym(2), blank() }),
         }),
-        RuleEntryTypeHidden
+        RuleEntryTypeAuxiliary
       },
     })));
   });
@@ -172,7 +172,7 @@ describe("expand_repeats", []() {
           i_token(10),
           choice({ i_sym(1), blank() }),
         }),
-        RuleEntryTypeHidden
+        RuleEntryTypeAuxiliary
       },
       {
         "rule0_repeat2",
@@ -180,7 +180,7 @@ describe("expand_repeats", []() {
           i_token(11),
           choice({ i_sym(2), blank() }),
         }),
-        RuleEntryTypeHidden
+        RuleEntryTypeAuxiliary
       },
     })));
   });
@@ -218,7 +218,7 @@ describe("expand_repeats", []() {
           i_token(10),
           choice({ i_sym(2), blank() }),
         }),
-        RuleEntryTypeHidden
+        RuleEntryTypeAuxiliary
       },
       {
         "rule1_repeat1",
@@ -226,7 +226,7 @@ describe("expand_repeats", []() {
           i_token(11),
           choice({ i_sym(3), blank() })
         }),
-        RuleEntryTypeHidden
+        RuleEntryTypeAuxiliary
       },
     })));
   });

spec/compiler/prepare_grammar/extract_tokens_spec.cc

@@ -24,18 +24,22 @@ describe("extract_tokens", []() {
             token(repeat(choice({ str("ef"), str("gh") }))),
           }),
         })),
+        RuleEntryTypeNamed,
       },
       {
         "rule_B",
         pattern("ij+"),
+        RuleEntryTypeNamed,
       },
       {
         "rule_C",
         choice({ str("kl"), blank() }),
+        RuleEntryTypeNamed,
       },
       {
         "rule_D",
-        repeat(i_sym(3))
+        repeat(i_sym(3)),
+        RuleEntryTypeNamed,
       }
     }, {}, {}});
 
@@ -96,14 +100,14 @@ describe("extract_tokens", []() {
       {
         "/cd*/",
         pattern("cd*"),
-        RuleEntryTypeHidden,
+        RuleEntryTypeAuxiliary,
       },
 
       // Rules marked as tokens become hidden rules.
       {
         "/(ef|gh)*/",
         repeat(choice({ str("ef"), str("gh") })),
-        RuleEntryTypeHidden,
+        RuleEntryTypeAuxiliary,
       },
 
       // This named rule was moved wholesale to the lexical grammar.
@@ -131,7 +135,8 @@ describe("extract_tokens", []() {
           str("ab"),
           i_sym(0),
           str("ab"),
-        })
+        }),
+        RuleEntryTypeNamed,
       },
     }, {}, {}});
 
@@ -159,15 +164,18 @@ describe("extract_tokens", []() {
     auto result = extract_tokens(InternedGrammar{{
       {
         "rule_A",
-        seq({ i_sym(1), str("ab") })
+        seq({ i_sym(1), str("ab") }),
+        RuleEntryTypeNamed,
       },
       {
         "rule_B",
-        str("cd")
+        str("cd"),
+        RuleEntryTypeNamed,
       },
       {
         "rule_C",
-        seq({ str("ef"), str("cd") })
+        seq({ str("ef"), str("cd") }),
+        RuleEntryTypeNamed,
       },
     }, {}, {}});
 
@@ -215,15 +223,18 @@ describe("extract_tokens", []() {
     auto result = extract_tokens(InternedGrammar{{
       {
         "rule_A",
-        str("ok")
+        str("ok"),
+        RuleEntryTypeNamed,
       },
       {
         "rule_B",
-        repeat(i_sym(0))
+        repeat(i_sym(0)),
+        RuleEntryTypeNamed,
       },
       {
         "rule_C",
-        repeat(seq({ i_sym(0), i_sym(0) }))
+        repeat(seq({ i_sym(0), i_sym(0) })),
+        RuleEntryTypeNamed,
       },
     }, { str(" ") }, { { Symbol(1), Symbol(2) } }});
 
@@ -238,7 +249,11 @@ describe("extract_tokens", []() {
   describe("handling ubiquitous tokens", [&]() {
     it("adds inline ubiquitous tokens to the lexical grammar's separators", [&]() {
       auto result = extract_tokens(InternedGrammar{{
-        { "rule_A", str("x") },
+        {
+          "rule_A",
+          str("x"),
+          RuleEntryTypeNamed,
+        },
       }, {
         pattern("\\s+"),
         str("y"),
@@ -256,9 +271,21 @@ describe("extract_tokens", []() {
 
     it("updates ubiquitous symbols according to the new symbol numbers", [&]() {
       auto result = extract_tokens(InternedGrammar{ {
-        { "rule_A", seq({ str("w"), str("x"), i_sym(1) }) },
-        { "rule_B", str("y") },
-        { "rule_C", str("z") },
+        {
+          "rule_A",
+          seq({ str("w"), str("x"), i_sym(1) }),
+          RuleEntryTypeNamed
+        },
+        {
+          "rule_B",
+          str("y"),
+          RuleEntryTypeNamed
+        },
+        {
+          "rule_C",
+          str("z"),
+          RuleEntryTypeNamed
+        },
       }, {
         i_sym(2),
       }, {}});
@@ -277,10 +304,12 @@ describe("extract_tokens", []() {
         {
           "rule_A",
           seq({ str("x"), i_sym(1) }),
+          RuleEntryTypeNamed,
         },
         {
           "rule_B",
-          seq({ str("y"), str("z") })
+          seq({ str("y"), str("z") }),
+          RuleEntryTypeNamed,
         },
       }, { i_sym(1) }, {}});
 
@@ -294,11 +323,13 @@ describe("extract_tokens", []() {
       auto result = extract_tokens(InternedGrammar{{
         {
           "rule_A",
-          str("x")
+          str("x"),
+          RuleEntryTypeNamed,
         },
         {
           "rule_B",
-          str("y")
+          str("y"),
+          RuleEntryTypeNamed,
         },
       }, { choice({ i_sym(1), blank() }) }, {}});
 

spec/compiler/prepare_grammar/intern_symbols_spec.cc

@@ -12,18 +12,30 @@ using prepare_grammar::intern_symbols;
 describe("intern_symbols", []() {
   it("replaces named symbols with numerically-indexed symbols", [&]() {
     Grammar grammar({
-      { "x", choice({ sym("y"), sym("z") }) },
-      { "y", sym("z") },
-      { "z", str("stuff") }
+      { "x", choice({ sym("y"), sym("_z") }) },
+      { "y", sym("_z") },
+      { "_z", str("stuff") }
     });
 
     auto result = intern_symbols(grammar);
 
     AssertThat(result.second, Equals((GrammarError *)nullptr));
-    AssertThat(result.first.rules, Equals(rule_list({
-      { "x", choice({ i_sym(1), i_sym(2) }) },
-      { "y", i_sym(2) },
-      { "z", str("stuff") },
+    AssertThat(result.first.rules, Equals(eq_vector<RuleEntry>({
+      {
+        "x",
+        choice({ i_sym(1), i_sym(2) }),
+        RuleEntryTypeNamed
+      },
+      {
+        "y",
+        i_sym(2),
+        RuleEntryTypeNamed,
+      },
+      {
+        "_z",
+        str("stuff"),
+        RuleEntryTypeHidden
+      },
     })));
   });