traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add accessor methods on Grammar

Browse source
This commit is contained in:
Max Brunsfeld 2014-06-09 21:05:25 -07:00
parent b4a34dd7c2
commit 54a555168d
10 changed files with 47 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

7
examples/grammars/golang.cc
View file

@ -3,14 +3,13 @@
namespace tree_sitter_examples {
using tree_sitter::Grammar;
using tree_sitter::GrammarOptions;
using namespace tree_sitter::rules;
static rule_ptr terminated(rule_ptr rule) {
return seq({ rule, sym("_terminator") });
}
extern const Grammar golang({
extern const Grammar golang = Grammar({
{ "program", seq({
sym("package_directive"),
repeat(sym("imports_block")),
@ -121,7 +120,5 @@ namespace tree_sitter_examples {
{ "_identifier", pattern("\\a[\\w_]*") },
{ "number", pattern("\\d+(\\.\\d+)?") },
{ "comment", pattern("//[^\n]*") },
}, GrammarOptions({
{ "comment" },
}));
}).ubiquitous_tokens({ "comment" });
}

7
examples/grammars/javascript.cc
View file

@ -5,7 +5,7 @@ namespace tree_sitter_examples {
using tree_sitter::Grammar;
using namespace tree_sitter::rules;
extern const Grammar javascript({
extern const Grammar javascript = Grammar({
{ "program", repeat(sym("statement")) },
// Statements
@ -181,8 +181,5 @@ namespace tree_sitter_examples {
{ "null", keyword("null") },
{ "true", keyword("true") },
{ "false", keyword("false") },
}, {
// ubiquitous_tokens
{ "comment" }
});
}).ubiquitous_tokens({ "comment" });
}

15
include/tree_sitter/compiler.h
View file

@ -25,19 +25,20 @@ namespace tree_sitter {
rule_ptr token(rule_ptr rule);
}
struct GrammarOptions {
std::vector<std::string> ubiquitous_tokens;
};
class Grammar {
protected:
std::vector<std::string> ubiquitous_tokens_;
const std::vector<std::pair<std::string, rules::rule_ptr>> rules_;
public:
Grammar(const std::vector<std::pair<std::string, rules::rule_ptr>> &rules);
Grammar(const std::vector<std::pair<std::string, rules::rule_ptr>> &rules, GrammarOptions options);
bool operator==(const Grammar &other) const;
std::string start_rule_name() const;
const rules::rule_ptr rule(const std::string &name) const;
const std::vector<std::pair<std::string, rules::rule_ptr>> rules;
const GrammarOptions options;
const std::vector<std::string> & ubiquitous_tokens() const;
const Grammar & ubiquitous_tokens(const std::vector<std::string> &ubiquitous_tokens);
const std::vector<std::pair<std::string, rules::rule_ptr>> & rules() const;
};
struct Conflict {

6
spec/compiler/prepare_grammar/intern_symbols_spec.cc
View file

@ -40,13 +40,11 @@ describe("interning symbols in a grammar", []() {
});
it("translates the grammar's optional 'ubiquitous_tokens' to numerical symbols", [&]() {
Grammar grammar({
auto grammar = Grammar({
{ "x", choice({ sym("y"), sym("z") }) },
{ "y", sym("z") },
{ "z", str("stuff") }
}, {
{ "z" }
});
}).ubiquitous_tokens({ "z" });
auto result = intern_symbols(grammar);

34
src/compiler/grammar.cc
View file

@ -4,22 +4,19 @@
namespace tree_sitter {
using std::string;
using std::ostream;
using std::pair;
using std::vector;
using rules::rule_ptr;
Grammar::Grammar(const std::vector<std::pair<std::string, rules::rule_ptr>> &rules) :
rules(rules),
options({}) {}
Grammar::Grammar(const std::vector<std::pair<std::string, rules::rule_ptr>> &rules, GrammarOptions options) :
rules(rules),
options(options) {}
rules_(rules) {}
bool Grammar::operator==(const Grammar &other) const {
if (other.rules.size() != rules.size()) return false;
if (other.rules_.size() != rules_.size()) return false;
for (size_t i = 0; i < rules.size(); i++) {
auto &pair = rules[i];
auto &other_pair = other.rules[i];
for (size_t i = 0; i < rules_.size(); i++) {
auto &pair = rules_[i];
auto &other_pair = other.rules_[i];
if (other_pair.first != pair.first) return false;
if (!other_pair.second->operator==(*pair.second)) return false;
}
@ -28,14 +25,14 @@ namespace tree_sitter {
}
string Grammar::start_rule_name() const {
return rules.front().first;
return rules_.front().first;
}
ostream& operator<<(ostream &stream, const Grammar &grammar) {
stream << string("#<grammar");
stream << string(" rules: {");
bool started = false;
for (auto pair : grammar.rules) {
for (auto pair : grammar.rules()) {
if (started) stream << string(", ");
stream << pair.first;
stream << string(" => ");
@ -59,4 +56,17 @@ namespace tree_sitter {
else
return stream << string("#<null>");
}
const vector<string> & Grammar::ubiquitous_tokens() const {
return ubiquitous_tokens_;
}
const Grammar & Grammar::ubiquitous_tokens(const vector<string> &ubiquitous_tokens) {
ubiquitous_tokens_ = ubiquitous_tokens;
return *this;
}
const vector<pair<string, rule_ptr>> & Grammar::rules() const {
return rules_;
}
}

2
src/compiler/prepare_grammar/expand_repeats.cc
View file

@ -53,7 +53,7 @@ namespace tree_sitter {
PreparedGrammar expand_repeats(const PreparedGrammar &grammar) {
vector<pair<string, rules::rule_ptr>> rules, aux_rules(grammar.aux_rules);
for (auto &pair : grammar.rules) {
for (auto &pair : grammar.rules()) {
ExpandRepeats expander(pair.first, aux_rules.size());
rules.push_back({ pair.first, expander.apply(pair.second) });
aux_rules.insert(aux_rules.end(), expander.aux_rules.begin(), expander.aux_rules.end());

2
src/compiler/prepare_grammar/expand_tokens.cc
View file

@ -48,7 +48,7 @@ namespace tree_sitter {
vector<pair<string, rule_ptr>> rules, aux_rules;
ExpandTokens expander;
for (auto &pair : grammar.rules) {
for (auto &pair : grammar.rules()) {
auto rule = expander.apply(pair.second);
if (expander.error)
return { PreparedGrammar(), expander.error };

4
src/compiler/prepare_grammar/extract_tokens.cc
View file

@ -101,8 +101,8 @@ namespace tree_sitter {
TokenExtractor extractor;
map<Symbol, Symbol> symbol_replacements;
for (size_t i = 0; i < input_grammar.rules.size(); i++) {
auto pair = input_grammar.rules[i];
for (size_t i = 0; i < input_grammar.rules().size(); i++) {
auto pair = input_grammar.rules()[i];
if (IsToken().apply(pair.second)) {
tokens.push_back(pair);
symbol_replacements.insert({

8
src/compiler/prepare_grammar/intern_symbols.cc
View file

@ -26,8 +26,8 @@ namespace tree_sitter {
public:
std::shared_ptr<rules::Symbol> symbol_for_rule_name(string rule_name) {
for (size_t i = 0; i < grammar.rules.size(); i++)
if (grammar.rules[i].first == rule_name)
for (size_t i = 0; i < grammar.rules().size(); i++)
if (grammar.rules()[i].first == rule_name)
return make_shared<rules::Symbol>(i);
return nullptr;
}
@ -49,7 +49,7 @@ namespace tree_sitter {
InternSymbols interner(grammar);
vector<pair<string, rule_ptr>> rules;
for (auto &pair : grammar.rules) {
for (auto &pair : grammar.rules()) {
auto new_rule = interner.apply(pair.second);
if (!interner.missing_rule_name.empty())
return missing_rule_error(interner.missing_rule_name);
@ -57,7 +57,7 @@ namespace tree_sitter {
}
vector<rules::Symbol> ubiquitous_tokens;
for (auto &name : grammar.options.ubiquitous_tokens) {
for (auto &name : grammar.ubiquitous_tokens()) {
auto token = interner.symbol_for_rule_name(name);
if (!token.get())
return missing_rule_error(name);

6
src/compiler/prepared_grammar.cc
View file

@ -29,13 +29,13 @@ namespace tree_sitter {
const rule_ptr & PreparedGrammar::rule(const Symbol &symbol) const {
return symbol.is_auxiliary() ?
aux_rules[symbol.index].second :
rules[symbol.index].second;
rules_[symbol.index].second;
}
const string & PreparedGrammar::rule_name(const Symbol &symbol) const {
return symbol.is_auxiliary() ?
aux_rules[symbol.index].first :
rules[symbol.index].first;
rules_[symbol.index].first;
}
bool PreparedGrammar::operator==(const PreparedGrammar &other) const {
@ -55,7 +55,7 @@ namespace tree_sitter {
stream << string(" rules: {");
bool started = false;
for (auto pair : grammar.rules) {
for (auto pair : grammar.rules()) {
if (started) stream << string(", ");
stream << pair.first;
stream << string(" => ");