traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Don’t allow duplicate rules when extracting tokens from grammars

Browse source
This commit is contained in:
Max Brunsfeld 2014-01-04 13:38:03 -08:00
parent 2afd8843dc
commit ed80d9cf52
2 changed files with 11 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

8
spec/compiler/grammar/prepare_grammar_spec.cpp
View file

@ -13,7 +13,10 @@ describe("preparing a grammar", []() {
character('b'),
seq({
sym("rule2"),
sym("rule3") }) }) }
sym("rule3") }),
seq({
character('a'),
character('b') }) }) }
}));
AssertThat(result.first, Equals(Grammar({
@ -21,7 +24,8 @@ describe("preparing a grammar", []() {
token("1"),
seq({
sym("rule2"),
sym("rule3") }) }) }
sym("rule3") }),
token("1") }) }
})));
AssertThat(result.second, Equals(Grammar("", {

6
src/compiler/grammar/extract_tokens.cpp
View file

@ -24,6 +24,9 @@ namespace tree_sitter {
}
string add_token(const rules::rule_ptr &rule) {
for (auto pair : tokens) {
if (*pair.second == *rule) return pair.first;
}
string name = to_string(tokens.size() + 1);
tokens.insert({ name, rule });
return name;
@ -47,7 +50,8 @@ namespace tree_sitter {
unordered_map<string, const rules::rule_ptr> rules;
for (auto pair : input_grammar.rules) {
rules.insert({ pair.first, extractor.apply(pair.second) });
auto new_rule = extractor.apply(pair.second);
rules.insert({ pair.first, new_rule });
}
return {