diff --git a/src/compiler/prepare_grammar/extract_tokens.cc b/src/compiler/prepare_grammar/extract_tokens.cc index 9d397fa2..ee4993b7 100644 --- a/src/compiler/prepare_grammar/extract_tokens.cc +++ b/src/compiler/prepare_grammar/extract_tokens.cc @@ -30,8 +30,7 @@ using rules::Symbol; using rules::SymbolOptionToken; using rules::SymbolOptionAuxToken; -class SymbolInliner : public rules::IdentityRuleFn { - map replacements; +class SymbolReplacer : public rules::IdentityRuleFn { using rules::IdentityRuleFn::apply_to; int new_index_for_symbol(const Symbol &symbol) { @@ -46,6 +45,8 @@ class SymbolInliner : public rules::IdentityRuleFn { rule_ptr apply_to(const Symbol *rule) { return replace_symbol(*rule).copy(); } public: + map replacements; + Symbol replace_symbol(const Symbol &rule) { if (rule.is_built_in()) return rule; @@ -55,9 +56,6 @@ class SymbolInliner : public rules::IdentityRuleFn { else return Symbol(new_index_for_symbol(rule), rule.options); } - - explicit SymbolInliner(const map &replacements) - : replacements(replacements) {} }; class TokenExtractor : public rules::IdentityRuleFn { @@ -93,57 +91,56 @@ class TokenExtractor : public rules::IdentityRuleFn { vector> tokens; }; -static const GrammarError *ubiq_token_err(const string &msg) { - return new GrammarError(GrammarErrorTypeInvalidUbiquitousToken, msg); +static tuple +ubiq_token_err(const rule_ptr rule) { + return make_tuple(SyntaxGrammar(), LexicalGrammar(), + new GrammarError(GrammarErrorTypeInvalidUbiquitousToken, + "Not a token: " + rule->to_string())); } tuple extract_tokens( const Grammar &grammar) { - vector> rules, tokens, aux_rules, aux_tokens; + vector> rules, tokens; vector separators; set ubiquitous_tokens; - map symbol_replacements; + SymbolReplacer symbol_replacer; TokenExtractor extractor; - for (size_t i = 0; i < grammar.rules().size(); i++) { - auto pair = grammar.rules()[i]; + size_t i = 0; + for (auto &pair : grammar.rules()) { if (is_token(pair.second)) { tokens.push_back(pair); - symbol_replacements.insert( - { Symbol(i), Symbol(tokens.size() - 1, SymbolOptionToken) }); + symbol_replacer.replacements.insert( + { Symbol(i), Symbol(tokens.size() - 1, SymbolOptionToken), }); } else { rules.push_back({ pair.first, extractor.apply(pair.second) }); } + i++; } - aux_tokens.insert(aux_tokens.end(), extractor.tokens.begin(), - extractor.tokens.end()); - - SymbolInliner inliner(symbol_replacements); for (auto &pair : rules) - pair.second = inliner.apply(pair.second); + pair.second = symbol_replacer.apply(pair.second); - for (auto rule : grammar.ubiquitous_tokens()) { + for (auto &rule : grammar.ubiquitous_tokens()) { if (is_token(rule)) { separators.push_back(rule); } else { auto sym = dynamic_pointer_cast(extractor.apply(rule)); if (!sym.get()) - return make_tuple(SyntaxGrammar(), LexicalGrammar(), - ubiq_token_err("Not a token: " + rule->to_string())); + return ubiq_token_err(rule); - Symbol symbol = inliner.replace_symbol(*sym); + Symbol symbol = symbol_replacer.replace_symbol(*sym); if (!symbol.is_token()) - return make_tuple(SyntaxGrammar(), LexicalGrammar(), - ubiq_token_err("Not a token: " + symbol.to_string())); + return ubiq_token_err(rule); ubiquitous_tokens.insert(symbol); } } - return make_tuple(SyntaxGrammar(rules, aux_rules, ubiquitous_tokens), - LexicalGrammar(tokens, aux_tokens, separators), nullptr); + return make_tuple(SyntaxGrammar(rules, {}, ubiquitous_tokens), + LexicalGrammar(tokens, extractor.tokens, separators), + nullptr); } } // namespace prepare_grammar