686dc0997c
The current pretty conservative approach is to avoid merging parse states which would cause a pair tokens to co-exist for the first time in any parse state, where the two tokens can start with the same character and at least one of the tokens can contain a character which is part of the grammar's separators.
19 lines
522 B
C++
19 lines
522 B
C++
#include "compiler/prepare_grammar/normalize_rules.h"
|
|
#include "compiler/prepare_grammar/extract_choices.h"
|
|
#include "compiler/rules/choice.h"
|
|
|
|
namespace tree_sitter {
|
|
namespace prepare_grammar {
|
|
|
|
LexicalGrammar normalize_rules(const LexicalGrammar &input_grammar) {
|
|
LexicalGrammar result(input_grammar);
|
|
|
|
for (LexicalVariable &variable : result.variables) {
|
|
variable.rule = rules::Choice::build(extract_choices(variable.rule));
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
} // namespace prepare_grammar
|
|
} // namespace tree_sitter
|