traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove inheritance link btwn PreparedGrammar and Grammar

Browse source
This commit is contained in:
Max Brunsfeld 2014-06-10 10:30:00 -07:00
parent 11acc7d087
commit e105f5cebc
11 changed files with 70 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

7
spec/compiler/build_tables/build_parse_table_spec.cc
View file

@ -10,14 +10,11 @@ using namespace build_tables;
START_TEST
describe("building parse tables", []() {
PreparedGrammar parse_grammar({
auto parse_grammar = PreparedGrammar({
{ "rule0", choice({ i_sym(1), i_sym(2) }) },
{ "rule1", i_token(0) },
{ "rule2", i_token(1) },
}, {}, PreparedGrammarOptions({
// ubiquitous_tokens
{ Symbol(2, SymbolOptionToken) }
}));
}, {}).ubiquitous_tokens({ Symbol(2, SymbolOptionToken) });
PreparedGrammar lex_grammar({
{ "token0", pattern("[a-c]") },

6
spec/compiler/prepare_grammar/extract_tokens_spec.cc
View file

@ -132,11 +132,9 @@ describe("extracting tokens from a grammar", []() {
{ "rule_A", str("ab") },
{ "rule_B", i_sym(0) },
{ "rule_C", i_sym(1) },
}, {}, PreparedGrammarOptions({
{ Symbol(0) }
})));
}, {}).ubiquitous_tokens({ Symbol(0) }));
AssertThat(result.first.options.ubiquitous_tokens, Equals(vector<Symbol>({
AssertThat(result.first.ubiquitous_tokens(), Equals(vector<Symbol>({
{ Symbol(0, SymbolOptionToken) }
})));
});

2
spec/compiler/prepare_grammar/intern_symbols_spec.cc
View file

@ -49,7 +49,7 @@ describe("interning symbols in a grammar", []() {
auto result = intern_symbols(grammar);
AssertThat(result.second, Equals((GrammarError *)nullptr));
AssertThat(result.first.options.ubiquitous_tokens, Equals(vector<Symbol>({
AssertThat(result.first.ubiquitous_tokens(), Equals(vector<Symbol>({
Symbol(2)
})));
});

2
src/compiler/build_tables/build_parse_table.cc
View file

@ -60,7 +60,7 @@ namespace tree_sitter {
}
void add_ubiquitous_token_actions(const ParseItemSet &item_set, ParseStateId state_id) {
for (const Symbol &symbol : grammar.options.ubiquitous_tokens) {
for (const Symbol &symbol : grammar.ubiquitous_tokens()) {
auto &actions = parse_table.states[state_id].actions;
if (actions.find(symbol) == actions.end())
parse_table.add_action(state_id, symbol, ParseAction::Shift(state_id, { 0 }));

2
src/compiler/generate_code/c_code.cc
View file

@ -266,7 +266,7 @@ namespace tree_sitter {
string ubiquitous_symbols_list() {
string result = "UBIQUITOUS_SYMBOLS = {\n";
for (auto &symbol : syntax_grammar.options.ubiquitous_tokens)
for (auto &symbol : syntax_grammar.ubiquitous_tokens())
result += indent("[" + symbol_id(symbol) + "] = 1,") + "\n";
return result + "};";
}

5
src/compiler/prepare_grammar/expand_repeats.cc
View file

@ -51,7 +51,7 @@ namespace tree_sitter {
};
PreparedGrammar expand_repeats(const PreparedGrammar &grammar) {
vector<pair<string, rules::rule_ptr>> rules, aux_rules(grammar.aux_rules);
vector<pair<string, rules::rule_ptr>> rules, aux_rules(grammar.aux_rules());
for (auto &pair : grammar.rules()) {
ExpandRepeats expander(pair.first, aux_rules.size());
@ -59,7 +59,8 @@ namespace tree_sitter {
aux_rules.insert(aux_rules.end(), expander.aux_rules.begin(), expander.aux_rules.end());
}
return PreparedGrammar(rules, aux_rules, grammar.options);
return PreparedGrammar(rules, aux_rules).
ubiquitous_tokens(grammar.ubiquitous_tokens());
}
}
}

6
src/compiler/prepare_grammar/expand_tokens.cc
View file

@ -55,14 +55,16 @@ namespace tree_sitter {
rules.push_back({ pair.first, rule });
}
for (auto &pair : grammar.aux_rules) {
for (auto &pair : grammar.aux_rules()) {
auto rule = expander.apply(pair.second);
if (expander.error)
return { PreparedGrammar(), expander.error };
aux_rules.push_back({ pair.first, rule });
}
return { PreparedGrammar(rules, aux_rules, grammar.options), nullptr };
return {
PreparedGrammar(rules, aux_rules).ubiquitous_tokens(grammar.ubiquitous_tokens()),
nullptr };
}
}
}

11
src/compiler/prepare_grammar/extract_tokens.cc
View file

@ -113,8 +113,8 @@ namespace tree_sitter {
}
}
for (size_t i = 0; i < input_grammar.aux_rules.size(); i++) {
auto pair = input_grammar.aux_rules[i];
for (size_t i = 0; i < input_grammar.aux_rules().size(); i++) {
auto pair = input_grammar.aux_rules()[i];
if (IsToken().apply(pair.second)) {
aux_tokens.push_back(pair);
symbol_replacements.insert({
@ -133,14 +133,11 @@ namespace tree_sitter {
pair.second = inliner.apply(pair.second);
for (auto &pair : aux_rules)
pair.second = inliner.apply(pair.second);
for (auto &symbol : input_grammar.options.ubiquitous_tokens)
for (auto &symbol : input_grammar.ubiquitous_tokens())
ubiquitous_tokens.push_back(inliner.replace_symbol(symbol));
PreparedGrammarOptions parse_options(input_grammar.options);
parse_options.ubiquitous_tokens = ubiquitous_tokens;
return {
PreparedGrammar(rules, aux_rules, parse_options),
PreparedGrammar(rules, aux_rules).ubiquitous_tokens(ubiquitous_tokens),
PreparedGrammar(tokens, aux_tokens)
};
}

2
src/compiler/prepare_grammar/intern_symbols.cc
View file

@ -65,7 +65,7 @@ namespace tree_sitter {
}
return {
PreparedGrammar(rules, {}, PreparedGrammarOptions({ ubiquitous_tokens })),
PreparedGrammar(rules, {}).ubiquitous_tokens(ubiquitous_tokens),
nullptr
};
}

62
src/compiler/prepared_grammar.cc
View file

@ -8,48 +8,72 @@ namespace tree_sitter {
using std::string;
using std::pair;
using std::ostream;
using std::vector;
using rules::rule_ptr;
using rules::Symbol;
PreparedGrammar::PreparedGrammar() : Grammar({}), aux_rules({}), options({}) {}
PreparedGrammar::PreparedGrammar() :
rules_({}),
aux_rules_({}),
ubiquitous_tokens_({}) {}
PreparedGrammar::PreparedGrammar(const std::vector<std::pair<std::string, rules::rule_ptr>> &rules,
const std::vector<std::pair<std::string, rules::rule_ptr>> &aux_rules) :
Grammar(rules),
aux_rules(aux_rules),
options({}) {}
PreparedGrammar::PreparedGrammar(const std::vector<std::pair<std::string, rules::rule_ptr>> &rules,
const std::vector<std::pair<std::string, rules::rule_ptr>> &aux_rules,
PreparedGrammarOptions options) :
Grammar(rules),
aux_rules(aux_rules),
options(options) {}
rules_(rules),
aux_rules_(aux_rules),
ubiquitous_tokens_({}) {}
const rule_ptr & PreparedGrammar::rule(const Symbol &symbol) const {
return symbol.is_auxiliary() ?
aux_rules[symbol.index].second :
aux_rules_[symbol.index].second :
rules_[symbol.index].second;
}
const string & PreparedGrammar::rule_name(const Symbol &symbol) const {
return symbol.is_auxiliary() ?
aux_rules[symbol.index].first :
aux_rules_[symbol.index].first :
rules_[symbol.index].first;
}
bool PreparedGrammar::operator==(const PreparedGrammar &other) const {
if (!Grammar::operator==(other)) return false;
if (other.aux_rules.size() != aux_rules.size()) return false;
for (size_t i = 0; i < aux_rules.size(); i++) {
auto &pair = aux_rules[i];
auto &other_pair = other.aux_rules[i];
if (other.rules_.size() != rules_.size()) return false;
for (size_t i = 0; i < rules_.size(); i++) {
auto &pair = rules_[i];
auto &other_pair = other.rules_[i];
if (other_pair.first != pair.first) return false;
if (!other_pair.second->operator==(*pair.second)) return false;
}
if (other.aux_rules_.size() != aux_rules_.size()) return false;
for (size_t i = 0; i < aux_rules_
.size(); i++) {
auto &pair = aux_rules_[i];
auto &other_pair = other.aux_rules_[i];
if (other_pair.first != pair.first) return false;
if (!other_pair.second->operator==(*pair.second)) return false;
}
return true;
}
const vector<pair<string, rule_ptr>> & PreparedGrammar::rules() const {
return rules_;
}
const vector<pair<string, rule_ptr>> & PreparedGrammar::aux_rules() const {
return aux_rules_;
}
const vector<Symbol> & PreparedGrammar::ubiquitous_tokens() const {
return ubiquitous_tokens_;
}
const PreparedGrammar & PreparedGrammar::ubiquitous_tokens(const vector<Symbol> &ubiquitous_tokens) {
ubiquitous_tokens_ = ubiquitous_tokens;
return *this;
}
ostream& operator<<(ostream &stream, const PreparedGrammar &grammar) {
stream << string("#<grammar");
@ -66,7 +90,7 @@ namespace tree_sitter {
stream << string(" aux_rules: {");
started = false;
for (auto pair : grammar.aux_rules) {
for (auto pair : grammar.aux_rules()) {
if (started) stream << string(", ");
stream << pair.first;
stream << string(" => ");

18
src/compiler/prepared_grammar.h
View file

@ -8,25 +8,23 @@
#include "compiler/rules/symbol.h"
namespace tree_sitter {
struct PreparedGrammarOptions {
std::vector<rules::Symbol> ubiquitous_tokens;
};
class PreparedGrammar {
const std::vector<std::pair<std::string, rules::rule_ptr>> rules_;
const std::vector<std::pair<std::string, rules::rule_ptr>> aux_rules_;
std::vector<rules::Symbol> ubiquitous_tokens_;
class PreparedGrammar : public Grammar {
public:
PreparedGrammar();
PreparedGrammar(const std::vector<std::pair<std::string, rules::rule_ptr>> &rules,
const std::vector<std::pair<std::string, rules::rule_ptr>> &aux_rules);
PreparedGrammar(const std::vector<std::pair<std::string, rules::rule_ptr>> &rules,
const std::vector<std::pair<std::string, rules::rule_ptr>> &aux_rules,
PreparedGrammarOptions options);
bool operator==(const PreparedGrammar &other) const;
const std::string & rule_name(const rules::Symbol &symbol) const;
const rules::rule_ptr & rule(const rules::Symbol &symbol) const;
const std::vector<std::pair<std::string, rules::rule_ptr>> aux_rules;
const PreparedGrammarOptions options;
const std::vector<rules::Symbol> & ubiquitous_tokens() const;
const PreparedGrammar & ubiquitous_tokens(const std::vector<rules::Symbol> &ubiquitous_tokens);
const std::vector<std::pair<std::string, rules::rule_ptr>> & rules() const;
const std::vector<std::pair<std::string, rules::rule_ptr>> & aux_rules() const;
};
std::ostream& operator<<(std::ostream &stream, const PreparedGrammar &grammar);