From 25791085c368e45a3ff3ffb093b7e600f42c6d9e Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Sun, 11 Oct 2015 16:55:30 -0700 Subject: [PATCH] Normalize lexical grammar rules before constructing lex table --- project.gyp | 1 + .../prepare_grammar/extract_choices_spec.cc | 33 ++++++++++++++++--- .../prepare_grammar/extract_choices.cc | 7 ++-- .../prepare_grammar/flatten_grammar.h | 5 +++ .../prepare_grammar/normalize_rules.cc | 19 +++++++++++ .../prepare_grammar/normalize_rules.h | 14 ++++++++ .../prepare_grammar/prepare_grammar.cc | 3 +- 7 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 src/compiler/prepare_grammar/normalize_rules.cc create mode 100644 src/compiler/prepare_grammar/normalize_rules.h diff --git a/project.gyp b/project.gyp index c6ae92a1..488f513e 100644 --- a/project.gyp +++ b/project.gyp @@ -36,6 +36,7 @@ 'src/compiler/prepare_grammar/flatten_grammar.cc', 'src/compiler/prepare_grammar/intern_symbols.cc', 'src/compiler/prepare_grammar/is_token.cc', + 'src/compiler/prepare_grammar/normalize_rules.cc', 'src/compiler/prepare_grammar/parse_regex.cc', 'src/compiler/prepare_grammar/prepare_grammar.cc', 'src/compiler/prepare_grammar/token_description.cc', diff --git a/spec/compiler/prepare_grammar/extract_choices_spec.cc b/spec/compiler/prepare_grammar/extract_choices_spec.cc index a25d19c2..4928ef2c 100644 --- a/spec/compiler/prepare_grammar/extract_choices_spec.cc +++ b/spec/compiler/prepare_grammar/extract_choices_spec.cc @@ -61,12 +61,37 @@ describe("extract_choices", []() { }))); }); - it("handles repeats", [&]() { - auto rule = repeat(choice({ sym("a"), sym("b") })); + it("does not move choices outside of repeats", [&]() { + auto rule = seq({ + choice({ sym("a"), sym("b") }), + repeat(seq({ + sym("c"), + choice({ + sym("d"), + sym("e"), + }), + sym("f"), + })), + sym("g"), + }); AssertThat(extract_choices(rule), Equals(rule_vector({ - repeat(sym("a")), - repeat(sym("b")), + seq({ + sym("a"), + repeat(choice({ + seq({ sym("c"), sym("d"), sym("f") }), + seq({ sym("c"), sym("e"), sym("f") }), + })), + sym("g"), + }), + seq({ + sym("b"), + repeat(choice({ + seq({ sym("c"), sym("d"), sym("f") }), + seq({ sym("c"), sym("e"), sym("f") }), + })), + sym("g"), + }), }))); }); }); diff --git a/src/compiler/prepare_grammar/extract_choices.cc b/src/compiler/prepare_grammar/extract_choices.cc index 40ecb093..043b5653 100644 --- a/src/compiler/prepare_grammar/extract_choices.cc +++ b/src/compiler/prepare_grammar/extract_choices.cc @@ -42,10 +42,9 @@ class ExtractChoices : public rules::RuleFn> { } vector apply_to(const rules::Repeat *rule) { - vector result; - for (auto element : apply(rule->content)) - result.push_back(make_shared(element)); - return result; + return vector({ + rules::Repeat::build(rules::Choice::build(apply(rule->content))), + }); } }; diff --git a/src/compiler/prepare_grammar/flatten_grammar.h b/src/compiler/prepare_grammar/flatten_grammar.h index 15d51120..e728af17 100644 --- a/src/compiler/prepare_grammar/flatten_grammar.h +++ b/src/compiler/prepare_grammar/flatten_grammar.h @@ -1,3 +1,6 @@ +#ifndef COMPILER_PREPARE_GRAMMAR_FLATTEN_GRAMMAR_H_ +#define COMPILER_PREPARE_GRAMMAR_FLATTEN_GRAMMAR_H_ + #include #include "tree_sitter/compiler.h" #include "compiler/syntax_grammar.h" @@ -11,3 +14,5 @@ SyntaxGrammar flatten_grammar(const InitialSyntaxGrammar &); } // namespace prepare_grammar } // namespace tree_sitter + +#endif // COMPILER_PREPARE_GRAMMAR_FLATTEN_GRAMMAR_H_ diff --git a/src/compiler/prepare_grammar/normalize_rules.cc b/src/compiler/prepare_grammar/normalize_rules.cc new file mode 100644 index 00000000..0e1da9fd --- /dev/null +++ b/src/compiler/prepare_grammar/normalize_rules.cc @@ -0,0 +1,19 @@ +#include "compiler/prepare_grammar/normalize_rules.h" +#include "compiler/prepare_grammar/extract_choices.h" +#include "compiler/rules/choice.h" + +namespace tree_sitter { +namespace prepare_grammar { + +LexicalGrammar normalize_rules(const LexicalGrammar &input_grammar) { + LexicalGrammar result(input_grammar); + + for (Variable &variable : result.variables) { + variable.rule = rules::Choice::build(extract_choices(variable.rule)); + } + + return result; +} + +} // namespace prepare_grammar +} // namespace tree_sitter diff --git a/src/compiler/prepare_grammar/normalize_rules.h b/src/compiler/prepare_grammar/normalize_rules.h new file mode 100644 index 00000000..4938e116 --- /dev/null +++ b/src/compiler/prepare_grammar/normalize_rules.h @@ -0,0 +1,14 @@ +#ifndef COMPILER_PREPARE_GRAMMAR_NORMALIZE_RULES_H_ +#define COMPILER_PREPARE_GRAMMAR_NORMALIZE_RULES_H_ + +#include "compiler/lexical_grammar.h" + +namespace tree_sitter { +namespace prepare_grammar { + +LexicalGrammar normalize_rules(const LexicalGrammar &); + +} // namespace prepare_grammar +} // namespace tree_sitter + +#endif // COMPILER_PREPARE_GRAMMAR_NORMALIZE_RULES_H_ diff --git a/src/compiler/prepare_grammar/prepare_grammar.cc b/src/compiler/prepare_grammar/prepare_grammar.cc index 03fcde30..ad80aaa7 100644 --- a/src/compiler/prepare_grammar/prepare_grammar.cc +++ b/src/compiler/prepare_grammar/prepare_grammar.cc @@ -4,6 +4,7 @@ #include "compiler/prepare_grammar/extract_tokens.h" #include "compiler/prepare_grammar/intern_symbols.h" #include "compiler/prepare_grammar/flatten_grammar.h" +#include "compiler/prepare_grammar/normalize_rules.h" #include "compiler/lexical_grammar.h" #include "compiler/prepare_grammar/initial_syntax_grammar.h" #include "compiler/syntax_grammar.h" @@ -39,7 +40,7 @@ tuple prepare_grammar( if (error) return make_tuple(SyntaxGrammar(), LexicalGrammar(), error); - return make_tuple(flatten_grammar(syntax_grammar), lex_grammar, nullptr); + return make_tuple(flatten_grammar(syntax_grammar), normalize_rules(lex_grammar), nullptr); } } // namespace prepare_grammar