Collapse rules that contain only a single token

src/compiler/grammar/extract_tokens.cpp

@@ -11,8 +11,22 @@ namespace tree_sitter {
     class TokenExtractor : rules::Visitor {
     public:
         rules::rule_ptr value;
+        size_t anonymous_token_count = 0;
         unordered_map<string, const rules::rule_ptr> tokens;
 
+        rules::rule_ptr initial_apply(string name, const rules::rule_ptr rule) {
+            auto result = apply(rule);
+            auto symbol = std::dynamic_pointer_cast<const rules::Symbol>(result);
+            if (symbol && *symbol != *rule) {
+                tokens.insert({ name, tokens[symbol->name] });
+                tokens.erase(symbol->name);
+                anonymous_token_count--;
+                return rules::rule_ptr();
+            } else {
+                return result;
+            }
+        }
+
         rules::rule_ptr apply(const rules::rule_ptr rule) {
             if (search_for_symbols(rule)) {
                 rule->accept(*this);
@@ -24,10 +38,10 @@ namespace tree_sitter {
         }
         
         string add_token(const rules::rule_ptr &rule) {
-            for (auto pair : tokens) {
-                if (*pair.second == *rule) return pair.first;
-            }
-            string name = to_string(tokens.size() + 1);
+            for (auto pair : tokens)
+                if (*pair.second == *rule)
+                    return pair.first;
+            string name = to_string(++anonymous_token_count);
             tokens.insert({ name, rule });
             return name;
         }
@@ -50,8 +64,10 @@ namespace tree_sitter {
         unordered_map<string, const rules::rule_ptr> rules;
 
         for (auto pair : input_grammar.rules) {
-            auto new_rule = extractor.apply(pair.second);
-            rules.insert({ pair.first, new_rule });
+            string name = pair.first;
+            auto new_rule = extractor.initial_apply(name, pair.second);
+            if (new_rule.get())
+                rules.insert({ name, new_rule });
         }
         
         return { 

src/compiler/grammar/grammar.cpp

@@ -16,7 +16,6 @@ namespace tree_sitter {
         rules(rules),
         start_rule_name(start_rule_name) {}
 
-
     const rules::rule_ptr Grammar::rule(const string &name) const {
         auto iter = rules.find(name);
         return (iter == rules.end()) ?
@@ -38,6 +37,7 @@ namespace tree_sitter {
         for (auto pair : rules) {
             auto other_pair = other.rules.find(pair.first);
             if (other_pair == other.rules.end()) return false;
+            auto orr = other_pair->second->to_string();;
             if (!other_pair->second->operator==(*pair.second)) return false;
         }
         return true;

src/compiler/lr/table_builder.cpp

@@ -31,7 +31,6 @@ namespace tree_sitter {
             void add_shift_actions(const ItemSet &item_set, size_t state_index) {
                 auto x = item_set.sym_transitions(grammar);
                 for (auto transition : x) {
-                    
                     rules::Symbol symbol = *transition.first;
                     ItemSet item_set = *transition.second;
                     size_t new_state_index = add_parse_state(item_set);

src/compiler/rules/rule.cpp

@@ -6,6 +6,10 @@ using std::string;
 
 namespace tree_sitter {
     namespace rules {
+        bool Rule::operator!=(const Rule &other) const {
+            return !this->operator==(other);
+        }
+        
         ostream& operator<<(ostream& stream, const Rule &rule) {
             return stream << rule.to_string();
         }

src/compiler/rules/rule.h

@@ -13,6 +13,7 @@ namespace tree_sitter {
         class Rule {
         public:
             virtual bool operator==(const Rule& other) const = 0;
+            bool operator!=(const Rule& other) const;
             virtual size_t hash_code() const = 0;
             virtual rule_ptr copy() const = 0;
             virtual std::string to_string() const = 0;