Separate concepts of hidden and auxiliary symbols

This way, we can hide auxiliary symbols from library users,
but still allow them to use hidden symbols

src/compiler/build_tables/item_set_transitions.cpp

@@ -24,7 +24,7 @@ namespace tree_sitter {
             for (auto transition : sym_transitions(item.rule)) {
                 Symbol rule = transition.first;
                 auto consumed_symbols = item.consumed_symbols;
-                consumed_symbols.push_back(rule.is_auxiliary);
+                consumed_symbols.push_back(rule.is_hidden());
                 ParseItem new_item(item.lhs, transition.second, consumed_symbols, item.lookahead_sym);
                 result.insert({ rule, item_set_closure(ParseItemSet({ new_item }), grammar) });
             }

src/compiler/build_tables/perform.cpp

@@ -14,8 +14,8 @@ namespace tree_sitter {
 
     namespace build_tables {
         static int NOT_FOUND = -1;
-        static Symbol START("start", true);
-        static Symbol END_OF_INPUT("end", true);
+        static Symbol START("start", rules::SymbolTypeAuxiliary);
+        static Symbol END_OF_INPUT("end", rules::SymbolTypeAuxiliary);
 
         class TableBuilder {
             const Grammar grammar;

src/compiler/generate_code/c_code.cpp

@@ -85,7 +85,7 @@ namespace tree_sitter {
                 {}
             
             string symbol_id(rules::Symbol symbol) {
-                if (symbol.is_auxiliary)
+                if (symbol.is_auxiliary())
                     return "ts_aux_" + symbol.name;
                 else
                     return "ts_symbol_" + symbol.name;

src/compiler/grammar.cpp

@@ -24,7 +24,7 @@ namespace tree_sitter {
         aux_rules(aux_rules) {}
 
     const rule_ptr Grammar::rule(const Symbol &symbol) const {
-        auto map = symbol.is_auxiliary ? aux_rules : rules;
+        auto map = symbol.is_auxiliary() ? aux_rules : rules;
         auto iter = map.find(symbol.name);
         if (iter != map.end())
             return iter->second;

src/compiler/prepare_grammar/expand_repeats.cpp

@@ -27,7 +27,7 @@ namespace tree_sitter {
             
             rule_ptr make_repeat_helper(string name, const rule_ptr &rule) {
                 return Choice::Build({
-                    Seq::Build({ rule, make_shared<Symbol>(name, true) }),
+                    Seq::Build({ rule, make_shared<Symbol>(name, SymbolTypeAuxiliary) }),
                     make_shared<Blank>() });
             }
             
@@ -35,7 +35,7 @@ namespace tree_sitter {
                 rule_ptr inner_rule = apply(rule->content);
                 string helper_rule_name = string("repeat_helper") + to_string(aux_rules.size() + 1);
                 aux_rules.insert({ helper_rule_name, make_repeat_helper(helper_rule_name, inner_rule) });
-                value = make_shared<Symbol>(helper_rule_name, true);
+                value = make_shared<Symbol>(helper_rule_name, SymbolTypeAuxiliary);
             }
             
             void visit(const Seq *rule) {

src/compiler/prepare_grammar/extract_tokens.cpp

@@ -37,7 +37,7 @@ namespace tree_sitter {
                     return value;
                 } else {
                     string token_name = add_token(rule);
-                    return make_shared<Symbol>(token_name, true);
+                    return make_shared<Symbol>(token_name, SymbolTypeAuxiliary);
                 }
             }
             

src/compiler/rules/rules.cpp

@@ -52,11 +52,11 @@ namespace tree_sitter {
         }
 
         rule_ptr sym(const string &name) {
-            return make_shared<Symbol>(name, false);
+            return make_shared<Symbol>(name);
         }
 
-        rule_ptr aux_sym(const string &name) {
-            return make_shared<Symbol>(name, true);
+        rule_ptr _sym(const string &name) {
+            return make_shared<Symbol>(name, SymbolTypeHidden);
         }
     }
 }

src/compiler/rules/symbol.cpp

@@ -1,13 +1,14 @@
 #include "symbol.h"
 #include "visitor.h"
+#include <map>
 
 using std::string;
 using std::hash;
 
 namespace tree_sitter  {
     namespace rules {
-        Symbol::Symbol(const std::string &name) : name(name), is_auxiliary(false) {};
-        Symbol::Symbol(const std::string &name, bool is_auxiliary) : name(name), is_auxiliary(is_auxiliary) {};
+        Symbol::Symbol(const std::string &name) : name(name), type(SymbolTypeNormal) {};
+        Symbol::Symbol(const std::string &name, SymbolType type) : name(name), type(type) {};
 
         bool Symbol::operator==(const Rule &rule) const {
             const Symbol *other = dynamic_cast<const Symbol *>(&rule);
@@ -15,11 +16,11 @@ namespace tree_sitter  {
         }
 
         bool Symbol::operator==(const Symbol &other) const {
-            return (other.name == name) && (other.is_auxiliary == is_auxiliary);
+            return (other.name == name) && (other.type == type);
         }
 
         size_t Symbol::hash_code() const {
-            return hash<string>()(name) ^ hash<bool>()(is_auxiliary);
+            return hash<string>()(name) ^ hash<short int>()(type);
         }
         
         rule_ptr Symbol::copy() const {
@@ -27,17 +28,30 @@ namespace tree_sitter  {
         }
         
         string Symbol::to_string() const {
-            return is_auxiliary ?
-                string("#<aux_sym '") + name + "'>" :
-                string("#<sym '") + name + "'>";
+            switch (type) {
+                case SymbolTypeNormal:
+                    return string("#<sym '") + name + "'>";
+                case SymbolTypeHidden:
+                    return string("#<hidden_sym '") + name + "'>";
+                case SymbolTypeAuxiliary:
+                    return string("#<aux_sym '") + name + "'>";
+            }
         }
         
         bool Symbol::operator<(const Symbol &other) const {
-            if (is_auxiliary < other.is_auxiliary) return true;
-            if (is_auxiliary > other.is_auxiliary) return false;
+            if (type < other.type) return true;
+            if (type > other.type) return false;
             return (name < other.name);
         }
         
+        bool Symbol::is_auxiliary() const {
+            return type == SymbolTypeAuxiliary;
+        }
+
+        bool Symbol::is_hidden() const {
+            return (type == SymbolTypeHidden || type == SymbolTypeAuxiliary);
+        }
+        
         void Symbol::accept(Visitor &visitor) const {
             visitor.visit(this);
         }

src/compiler/rules/symbol.h

@@ -6,10 +6,16 @@
 
 namespace tree_sitter  {
     namespace rules {
+        typedef enum {
+            SymbolTypeNormal,
+            SymbolTypeHidden,
+            SymbolTypeAuxiliary
+        } SymbolType; 
+        
         class Symbol : public Rule {
         public:
             Symbol(const std::string &name);
-            Symbol(const std::string &name, bool is_auxiliary);
+            Symbol(const std::string &name, SymbolType type);
 
             bool operator==(const Rule& other) const;
             bool operator==(const Symbol &other) const;
@@ -20,8 +26,11 @@ namespace tree_sitter  {
             void accept(Visitor &visitor) const;
             bool operator<(const Symbol &other) const;
 
+            bool is_hidden() const;
+            bool is_auxiliary() const;
+
             std::string name;
-            bool is_auxiliary;
+            SymbolType type;
         };
     }
 }