traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Normalize lexical grammar rules before constructing lex table

Browse source
This commit is contained in:
Max Brunsfeld 2015-10-11 16:55:30 -07:00
parent 3ee58461d7
commit 25791085c3
7 changed files with 73 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

7
src/compiler/prepare_grammar/extract_choices.cc
View file

@ -42,10 +42,9 @@ class ExtractChoices : public rules::RuleFn<vector<rule_ptr>> {
}
vector<rule_ptr> apply_to(const rules::Repeat *rule) {
vector<rule_ptr> result;
for (auto element : apply(rule->content))
result.push_back(make_shared<rules::Repeat>(element));
return result;
return vector<rule_ptr>({
rules::Repeat::build(rules::Choice::build(apply(rule->content))),
});
}
};

5
src/compiler/prepare_grammar/flatten_grammar.h
View file

@ -1,3 +1,6 @@
#ifndef COMPILER_PREPARE_GRAMMAR_FLATTEN_GRAMMAR_H_
#define COMPILER_PREPARE_GRAMMAR_FLATTEN_GRAMMAR_H_
#include <string>
#include "tree_sitter/compiler.h"
#include "compiler/syntax_grammar.h"
@ -11,3 +14,5 @@ SyntaxGrammar flatten_grammar(const InitialSyntaxGrammar &);
} // namespace prepare_grammar
} // namespace tree_sitter
#endif // COMPILER_PREPARE_GRAMMAR_FLATTEN_GRAMMAR_H_

19
src/compiler/prepare_grammar/normalize_rules.cc Normal file
View file

@ -0,0 +1,19 @@
#include "compiler/prepare_grammar/normalize_rules.h"
#include "compiler/prepare_grammar/extract_choices.h"
#include "compiler/rules/choice.h"
namespace tree_sitter {
namespace prepare_grammar {
LexicalGrammar normalize_rules(const LexicalGrammar &input_grammar) {
LexicalGrammar result(input_grammar);
for (Variable &variable : result.variables) {
variable.rule = rules::Choice::build(extract_choices(variable.rule));
}
return result;
}
} // namespace prepare_grammar
} // namespace tree_sitter

14
src/compiler/prepare_grammar/normalize_rules.h Normal file
View file

@ -0,0 +1,14 @@
#ifndef COMPILER_PREPARE_GRAMMAR_NORMALIZE_RULES_H_
#define COMPILER_PREPARE_GRAMMAR_NORMALIZE_RULES_H_
#include "compiler/lexical_grammar.h"
namespace tree_sitter {
namespace prepare_grammar {
LexicalGrammar normalize_rules(const LexicalGrammar &);
} // namespace prepare_grammar
} // namespace tree_sitter
#endif // COMPILER_PREPARE_GRAMMAR_NORMALIZE_RULES_H_

3
src/compiler/prepare_grammar/prepare_grammar.cc
View file

@ -4,6 +4,7 @@
#include "compiler/prepare_grammar/extract_tokens.h"
#include "compiler/prepare_grammar/intern_symbols.h"
#include "compiler/prepare_grammar/flatten_grammar.h"
#include "compiler/prepare_grammar/normalize_rules.h"
#include "compiler/lexical_grammar.h"
#include "compiler/prepare_grammar/initial_syntax_grammar.h"
#include "compiler/syntax_grammar.h"
@ -39,7 +40,7 @@ tuple<SyntaxGrammar, LexicalGrammar, const GrammarError *> prepare_grammar(
if (error)
return make_tuple(SyntaxGrammar(), LexicalGrammar(), error);
return make_tuple(flatten_grammar(syntax_grammar), lex_grammar, nullptr);
return make_tuple(flatten_grammar(syntax_grammar), normalize_rules(lex_grammar), nullptr);
}
} // namespace prepare_grammar