diff --git a/spec/compiler/grammar/prepare_grammar_spec.cpp b/spec/compiler/grammar/prepare_grammar_spec.cpp index f2d2a89f..9a7e345b 100644 --- a/spec/compiler/grammar/prepare_grammar_spec.cpp +++ b/spec/compiler/grammar/prepare_grammar_spec.cpp @@ -13,7 +13,10 @@ describe("preparing a grammar", []() { character('b'), seq({ sym("rule2"), - sym("rule3") }) }) } + sym("rule3") }), + seq({ + character('a'), + character('b') }) }) } })); AssertThat(result.first, Equals(Grammar({ @@ -21,7 +24,8 @@ describe("preparing a grammar", []() { token("1"), seq({ sym("rule2"), - sym("rule3") }) }) } + sym("rule3") }), + token("1") }) } }))); AssertThat(result.second, Equals(Grammar("", { diff --git a/src/compiler/grammar/extract_tokens.cpp b/src/compiler/grammar/extract_tokens.cpp index 6c71073a..7f13bb81 100644 --- a/src/compiler/grammar/extract_tokens.cpp +++ b/src/compiler/grammar/extract_tokens.cpp @@ -24,6 +24,9 @@ namespace tree_sitter { } string add_token(const rules::rule_ptr &rule) { + for (auto pair : tokens) { + if (*pair.second == *rule) return pair.first; + } string name = to_string(tokens.size() + 1); tokens.insert({ name, rule }); return name; @@ -47,7 +50,8 @@ namespace tree_sitter { unordered_map rules; for (auto pair : input_grammar.rules) { - rules.insert({ pair.first, extractor.apply(pair.second) }); + auto new_rule = extractor.apply(pair.second); + rules.insert({ pair.first, new_rule }); } return {