traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Handle inline ubiquitous that are used elsewhere in the grammar

Browse source
This commit is contained in:
Max Brunsfeld 2015-10-26 17:19:37 -07:00
parent b7e0cb1fc6
commit b61b27f22f
2 changed files with 26 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

13
spec/compiler/prepare_grammar/extract_tokens_spec.cc
View file

@ -159,6 +159,19 @@ describe("extract_tokens", []() {
AssertThat(get<0>(result).ubiquitous_tokens, IsEmpty());
});
it("handles inline ubiquitous tokens that match tokens in the grammar", [&]() {
auto result = extract_tokens(InternedGrammar{{
Variable("rule_A", VariableTypeNamed, str("x")),
Variable("rule_B", VariableTypeNamed, str("y")),
}, {
str("y"),
}, {}});
AssertThat(get<2>(result), Equals<const GrammarError *>(nullptr));
AssertThat(get<1>(result).separators.size(), Equals<size_t>(0));
AssertThat(get<0>(result).ubiquitous_tokens, Equals(set<Symbol>({ Symbol(1, true) })));
});
it("updates ubiquitous symbols according to the new symbol numbers", [&]() {
auto result = extract_tokens(InternedGrammar{{
Variable("rule_A", VariableTypeNamed, seq({ str("w"), str("x"), i_sym(1) })),

13
src/compiler/prepare_grammar/extract_tokens.cc
View file

@ -154,6 +154,19 @@ tuple<InitialSyntaxGrammar, LexicalGrammar, const GrammarError *> extract_tokens
* lexical grammar's separator rules.
*/
for (const rule_ptr &rule : grammar.ubiquitous_tokens) {
int i = 0;
bool used_elsewhere_in_grammar = false;
for (const Variable &variable : lexical_grammar.variables) {
if (variable.rule->operator==(*rule)) {
syntax_grammar.ubiquitous_tokens.insert(Symbol(i, true));
used_elsewhere_in_grammar = true;
}
i++;
}
if (used_elsewhere_in_grammar)
continue;
if (is_token(rule)) {
lexical_grammar.separators.push_back(rule);
continue;