From 54a555168d1ea62856a820cb0d42d4f640de8992 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 9 Jun 2014 21:05:25 -0700 Subject: [PATCH] Add accessor methods on Grammar --- examples/grammars/golang.cc | 7 ++-- examples/grammars/javascript.cc | 7 ++-- include/tree_sitter/compiler.h | 15 ++++---- .../prepare_grammar/intern_symbols_spec.cc | 6 ++-- src/compiler/grammar.cc | 34 ++++++++++++------- .../prepare_grammar/expand_repeats.cc | 2 +- src/compiler/prepare_grammar/expand_tokens.cc | 2 +- .../prepare_grammar/extract_tokens.cc | 4 +-- .../prepare_grammar/intern_symbols.cc | 8 ++--- src/compiler/prepared_grammar.cc | 6 ++-- 10 files changed, 47 insertions(+), 44 deletions(-) diff --git a/examples/grammars/golang.cc b/examples/grammars/golang.cc index 2645f670..75ae494c 100644 --- a/examples/grammars/golang.cc +++ b/examples/grammars/golang.cc @@ -3,14 +3,13 @@ namespace tree_sitter_examples { using tree_sitter::Grammar; - using tree_sitter::GrammarOptions; using namespace tree_sitter::rules; static rule_ptr terminated(rule_ptr rule) { return seq({ rule, sym("_terminator") }); } - extern const Grammar golang({ + extern const Grammar golang = Grammar({ { "program", seq({ sym("package_directive"), repeat(sym("imports_block")), @@ -121,7 +120,5 @@ namespace tree_sitter_examples { { "_identifier", pattern("\\a[\\w_]*") }, { "number", pattern("\\d+(\\.\\d+)?") }, { "comment", pattern("//[^\n]*") }, - }, GrammarOptions({ - { "comment" }, - })); + }).ubiquitous_tokens({ "comment" }); } diff --git a/examples/grammars/javascript.cc b/examples/grammars/javascript.cc index b04e42d3..ea3292c6 100644 --- a/examples/grammars/javascript.cc +++ b/examples/grammars/javascript.cc @@ -5,7 +5,7 @@ namespace tree_sitter_examples { using tree_sitter::Grammar; using namespace tree_sitter::rules; - extern const Grammar javascript({ + extern const Grammar javascript = Grammar({ { "program", repeat(sym("statement")) }, // Statements @@ -181,8 +181,5 @@ namespace tree_sitter_examples { { "null", keyword("null") }, { "true", keyword("true") }, { "false", keyword("false") }, - }, { - // ubiquitous_tokens - { "comment" } - }); + }).ubiquitous_tokens({ "comment" }); } diff --git a/include/tree_sitter/compiler.h b/include/tree_sitter/compiler.h index eeb73660..f96b86c0 100644 --- a/include/tree_sitter/compiler.h +++ b/include/tree_sitter/compiler.h @@ -25,19 +25,20 @@ namespace tree_sitter { rule_ptr token(rule_ptr rule); } - struct GrammarOptions { - std::vector ubiquitous_tokens; - }; - class Grammar { + protected: + std::vector ubiquitous_tokens_; + const std::vector> rules_; + public: Grammar(const std::vector> &rules); - Grammar(const std::vector> &rules, GrammarOptions options); bool operator==(const Grammar &other) const; std::string start_rule_name() const; const rules::rule_ptr rule(const std::string &name) const; - const std::vector> rules; - const GrammarOptions options; + + const std::vector & ubiquitous_tokens() const; + const Grammar & ubiquitous_tokens(const std::vector &ubiquitous_tokens); + const std::vector> & rules() const; }; struct Conflict { diff --git a/spec/compiler/prepare_grammar/intern_symbols_spec.cc b/spec/compiler/prepare_grammar/intern_symbols_spec.cc index 99887360..ca2711d3 100644 --- a/spec/compiler/prepare_grammar/intern_symbols_spec.cc +++ b/spec/compiler/prepare_grammar/intern_symbols_spec.cc @@ -40,13 +40,11 @@ describe("interning symbols in a grammar", []() { }); it("translates the grammar's optional 'ubiquitous_tokens' to numerical symbols", [&]() { - Grammar grammar({ + auto grammar = Grammar({ { "x", choice({ sym("y"), sym("z") }) }, { "y", sym("z") }, { "z", str("stuff") } - }, { - { "z" } - }); + }).ubiquitous_tokens({ "z" }); auto result = intern_symbols(grammar); diff --git a/src/compiler/grammar.cc b/src/compiler/grammar.cc index 6d6d4930..5c5101cf 100644 --- a/src/compiler/grammar.cc +++ b/src/compiler/grammar.cc @@ -4,22 +4,19 @@ namespace tree_sitter { using std::string; using std::ostream; + using std::pair; + using std::vector; using rules::rule_ptr; Grammar::Grammar(const std::vector> &rules) : - rules(rules), - options({}) {} - - Grammar::Grammar(const std::vector> &rules, GrammarOptions options) : - rules(rules), - options(options) {} + rules_(rules) {} bool Grammar::operator==(const Grammar &other) const { - if (other.rules.size() != rules.size()) return false; + if (other.rules_.size() != rules_.size()) return false; - for (size_t i = 0; i < rules.size(); i++) { - auto &pair = rules[i]; - auto &other_pair = other.rules[i]; + for (size_t i = 0; i < rules_.size(); i++) { + auto &pair = rules_[i]; + auto &other_pair = other.rules_[i]; if (other_pair.first != pair.first) return false; if (!other_pair.second->operator==(*pair.second)) return false; } @@ -28,14 +25,14 @@ namespace tree_sitter { } string Grammar::start_rule_name() const { - return rules.front().first; + return rules_.front().first; } ostream& operator<<(ostream &stream, const Grammar &grammar) { stream << string("# "); @@ -59,4 +56,17 @@ namespace tree_sitter { else return stream << string("#"); } + + const vector & Grammar::ubiquitous_tokens() const { + return ubiquitous_tokens_; + } + + const Grammar & Grammar::ubiquitous_tokens(const vector &ubiquitous_tokens) { + ubiquitous_tokens_ = ubiquitous_tokens; + return *this; + } + + const vector> & Grammar::rules() const { + return rules_; + } } diff --git a/src/compiler/prepare_grammar/expand_repeats.cc b/src/compiler/prepare_grammar/expand_repeats.cc index 304be59f..d812b2bb 100644 --- a/src/compiler/prepare_grammar/expand_repeats.cc +++ b/src/compiler/prepare_grammar/expand_repeats.cc @@ -53,7 +53,7 @@ namespace tree_sitter { PreparedGrammar expand_repeats(const PreparedGrammar &grammar) { vector> rules, aux_rules(grammar.aux_rules); - for (auto &pair : grammar.rules) { + for (auto &pair : grammar.rules()) { ExpandRepeats expander(pair.first, aux_rules.size()); rules.push_back({ pair.first, expander.apply(pair.second) }); aux_rules.insert(aux_rules.end(), expander.aux_rules.begin(), expander.aux_rules.end()); diff --git a/src/compiler/prepare_grammar/expand_tokens.cc b/src/compiler/prepare_grammar/expand_tokens.cc index 765a7941..9bb10c20 100644 --- a/src/compiler/prepare_grammar/expand_tokens.cc +++ b/src/compiler/prepare_grammar/expand_tokens.cc @@ -48,7 +48,7 @@ namespace tree_sitter { vector> rules, aux_rules; ExpandTokens expander; - for (auto &pair : grammar.rules) { + for (auto &pair : grammar.rules()) { auto rule = expander.apply(pair.second); if (expander.error) return { PreparedGrammar(), expander.error }; diff --git a/src/compiler/prepare_grammar/extract_tokens.cc b/src/compiler/prepare_grammar/extract_tokens.cc index ac17fd8f..917e5189 100644 --- a/src/compiler/prepare_grammar/extract_tokens.cc +++ b/src/compiler/prepare_grammar/extract_tokens.cc @@ -101,8 +101,8 @@ namespace tree_sitter { TokenExtractor extractor; map symbol_replacements; - for (size_t i = 0; i < input_grammar.rules.size(); i++) { - auto pair = input_grammar.rules[i]; + for (size_t i = 0; i < input_grammar.rules().size(); i++) { + auto pair = input_grammar.rules()[i]; if (IsToken().apply(pair.second)) { tokens.push_back(pair); symbol_replacements.insert({ diff --git a/src/compiler/prepare_grammar/intern_symbols.cc b/src/compiler/prepare_grammar/intern_symbols.cc index 00da7b77..3259e470 100644 --- a/src/compiler/prepare_grammar/intern_symbols.cc +++ b/src/compiler/prepare_grammar/intern_symbols.cc @@ -26,8 +26,8 @@ namespace tree_sitter { public: std::shared_ptr symbol_for_rule_name(string rule_name) { - for (size_t i = 0; i < grammar.rules.size(); i++) - if (grammar.rules[i].first == rule_name) + for (size_t i = 0; i < grammar.rules().size(); i++) + if (grammar.rules()[i].first == rule_name) return make_shared(i); return nullptr; } @@ -49,7 +49,7 @@ namespace tree_sitter { InternSymbols interner(grammar); vector> rules; - for (auto &pair : grammar.rules) { + for (auto &pair : grammar.rules()) { auto new_rule = interner.apply(pair.second); if (!interner.missing_rule_name.empty()) return missing_rule_error(interner.missing_rule_name); @@ -57,7 +57,7 @@ namespace tree_sitter { } vector ubiquitous_tokens; - for (auto &name : grammar.options.ubiquitous_tokens) { + for (auto &name : grammar.ubiquitous_tokens()) { auto token = interner.symbol_for_rule_name(name); if (!token.get()) return missing_rule_error(name); diff --git a/src/compiler/prepared_grammar.cc b/src/compiler/prepared_grammar.cc index 5e877e6c..a321b289 100644 --- a/src/compiler/prepared_grammar.cc +++ b/src/compiler/prepared_grammar.cc @@ -29,13 +29,13 @@ namespace tree_sitter { const rule_ptr & PreparedGrammar::rule(const Symbol &symbol) const { return symbol.is_auxiliary() ? aux_rules[symbol.index].second : - rules[symbol.index].second; + rules_[symbol.index].second; } const string & PreparedGrammar::rule_name(const Symbol &symbol) const { return symbol.is_auxiliary() ? aux_rules[symbol.index].first : - rules[symbol.index].first; + rules_[symbol.index].first; } bool PreparedGrammar::operator==(const PreparedGrammar &other) const { @@ -55,7 +55,7 @@ namespace tree_sitter { stream << string(" rules: {"); bool started = false; - for (auto pair : grammar.rules) { + for (auto pair : grammar.rules()) { if (started) stream << string(", "); stream << pair.first; stream << string(" => ");