In compiler, distinguish between anonymous tokens and hidden rules

src/compiler/rules/built_in_symbols.cc

@@ -4,11 +4,11 @@ namespace tree_sitter {
 namespace rules {
 
 Symbol END_OF_INPUT() {
-  return Symbol(-1, SymbolOptionToken);
+  return Symbol(-1, true);
 }
 
 Symbol ERROR() {
-  return Symbol(-2, SymbolOptionToken);
+  return Symbol(-2, true);
 }
 
 Symbol START() {

src/compiler/rules/rules.cc

@@ -52,7 +52,7 @@ rule_ptr pattern(const string &value) {
 }
 
 rule_ptr str(const string &value) {
-  return token(prec(1, make_shared<rules::String>(value)));
+  return make_shared<rules::String>(value);
 }
 
 rule_ptr err(const rule_ptr &rule) {

src/compiler/rules/symbol.cc

@@ -10,16 +10,12 @@ using std::string;
 using std::to_string;
 using std::hash;
 
-SymbolOption SymbolOptionAuxToken =
-  SymbolOption(SymbolOptionToken | SymbolOptionAuxiliary);
+Symbol::Symbol(int index) : index(index), is_token(false) {}
 
-Symbol::Symbol(int index) : index(index), options(SymbolOption(0)) {}
-
-Symbol::Symbol(int index, SymbolOption options)
-    : index(index), options(options) {}
+Symbol::Symbol(int index, bool is_token) : index(index), is_token(is_token) {}
 
 bool Symbol::operator==(const Symbol &other) const {
-  return (other.index == index) && (other.options == options);
+  return (other.index == index) && (other.is_token == is_token);
 }
 
 bool Symbol::operator==(const Rule &rule) const {
@@ -28,7 +24,7 @@ bool Symbol::operator==(const Rule &rule) const {
 }
 
 size_t Symbol::hash_code() const {
-  return hash<int>()(index) ^ hash<int16_t>()(options);
+  return hash<int>()(index) ^ hash<bool>()(is_token);
 }
 
 rule_ptr Symbol::copy() const {
@@ -36,31 +32,22 @@ rule_ptr Symbol::copy() const {
 }
 
 string Symbol::to_string() const {
-  string name = (options & SymbolOptionAuxiliary) ? "aux_" : "";
-  name += (options & SymbolOptionToken) ? "token" : "sym";
+  string name = is_token ? "token" : "sym";
   return "(" + name + " " + std::to_string(index) + ")";
 }
 
 bool Symbol::operator<(const Symbol &other) const {
-  if (options < other.options)
+  if (!is_token && other.is_token)
     return true;
-  if (options > other.options)
+  if (is_token && !other.is_token)
     return false;
   return (index < other.index);
 }
 
-bool Symbol::is_token() const {
-  return options & SymbolOptionToken;
-}
-
 bool Symbol::is_built_in() const {
   return index < 0;
 }
 
-bool Symbol::is_auxiliary() const {
-  return options & SymbolOptionAuxiliary;
-}
-
 void Symbol::accept(Visitor *visitor) const {
   visitor->visit(this);
 }

src/compiler/rules/symbol.h

@@ -7,17 +7,10 @@
 namespace tree_sitter {
 namespace rules {
 
-typedef enum {
-  SymbolOptionToken = 1 << 0,
-  SymbolOptionAuxiliary = 1 << 1,
-} SymbolOption;
-
-extern SymbolOption SymbolOptionAuxToken;
-
 class Symbol : public Rule {
  public:
   explicit Symbol(int index);
-  Symbol(int index, SymbolOption options);
+  Symbol(int index, bool is_token);
 
   bool operator==(const Symbol &other) const;
   bool operator==(const Rule &other) const;
@@ -28,12 +21,10 @@ class Symbol : public Rule {
   void accept(Visitor *visitor) const;
 
   bool operator<(const Symbol &other) const;
-  bool is_token() const;
   bool is_built_in() const;
-  bool is_auxiliary() const;
 
   int index;
-  SymbolOption options;
+  bool is_token;
 };
 
 }  // namespace rules