From ed80d9cf52fa9d243a025d2f22eb8c6e8238709f Mon Sep 17 00:00:00 2001
From: Max Brunsfeld <maxbrunsfeld@gmail.com>
Date: Sat, 4 Jan 2014 13:38:03 -0800
Subject: [PATCH] =?UTF-8?q?Don=E2=80=99t=20allow=20duplicate=20rules=20whe?=
 =?UTF-8?q?n=20extracting=20tokens=20from=20grammars?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 spec/compiler/grammar/prepare_grammar_spec.cpp | 8 ++++++--
 src/compiler/grammar/extract_tokens.cpp        | 6 +++++-
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/spec/compiler/grammar/prepare_grammar_spec.cpp b/spec/compiler/grammar/prepare_grammar_spec.cpp
index f2d2a89f..9a7e345b 100644
--- a/spec/compiler/grammar/prepare_grammar_spec.cpp
+++ b/spec/compiler/grammar/prepare_grammar_spec.cpp
@@ -13,7 +13,10 @@ describe("preparing a grammar", []() {
                 character('b'),
                 seq({
                     sym("rule2"),
-                    sym("rule3") }) }) }
+                    sym("rule3") }),
+                seq({
+                    character('a'),
+                    character('b') }) }) }
         }));
         
         AssertThat(result.first, Equals(Grammar({
@@ -21,7 +24,8 @@ describe("preparing a grammar", []() {
                 token("1"),
                 seq({
                     sym("rule2"),
-                    sym("rule3") }) }) }
+                    sym("rule3") }),
+                token("1") }) }
         })));
 
         AssertThat(result.second, Equals(Grammar("", {
diff --git a/src/compiler/grammar/extract_tokens.cpp b/src/compiler/grammar/extract_tokens.cpp
index 6c71073a..7f13bb81 100644
--- a/src/compiler/grammar/extract_tokens.cpp
+++ b/src/compiler/grammar/extract_tokens.cpp
@@ -24,6 +24,9 @@ namespace tree_sitter {
         }
         
         string add_token(const rules::rule_ptr &rule) {
+            for (auto pair : tokens) {
+                if (*pair.second == *rule) return pair.first;
+            }
             string name = to_string(tokens.size() + 1);
             tokens.insert({ name, rule });
             return name;
@@ -47,7 +50,8 @@ namespace tree_sitter {
         unordered_map<string, const rules::rule_ptr> rules;
 
         for (auto pair : input_grammar.rules) {
-            rules.insert({ pair.first, extractor.apply(pair.second) });
+            auto new_rule = extractor.apply(pair.second);
+            rules.insert({ pair.first, new_rule });
         }
         
         return {