traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Store grammars' rules in vectors, not maps

Browse source 
This way, we can keep of the order in which the rules
were given, for resolving reduce/reduce conflicts.
This commit is contained in:
Max Brunsfeld 2014-03-24 13:05:04 -07:00
parent 671f1a1ddc
commit 48baf056b4
18 changed files with 525 additions and 533 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/compiler/build_tables/build_tables.cc
View file

@ -153,7 +153,7 @@ namespace tree_sitter {
lex_grammar(lex_grammar) {}
pair<ParseTable, LexTable> build() {
auto start_symbol = make_shared<Symbol>(grammar.start_rule_name);
auto start_symbol = make_shared<Symbol>(grammar.start_rule_name());
ParseItem item(rules::START, start_symbol, {}, rules::END_OF_INPUT);
ParseItemSet item_set = item_set_closure(ParseItemSet({ item }), grammar);
add_parse_state(item_set);

18
src/compiler/grammar.cc
View file

@ -3,26 +3,28 @@
namespace tree_sitter {
using std::string;
using std::map;
using std::ostream;
using rules::rule_ptr;
Grammar::Grammar(std::string start_rule_name, const map<const string, const rule_ptr> &rules) :
start_rule_name(start_rule_name),
Grammar::Grammar(const std::vector<std::pair<std::string, rules::rule_ptr>> &rules) :
rules(rules) {}
bool Grammar::operator==(const Grammar &other) const {
if (other.start_rule_name != start_rule_name) return false;
if (other.rules.size() != rules.size()) return false;
for (auto pair : rules) {
auto other_pair = other.rules.find(pair.first);
if (other_pair == other.rules.end()) return false;
if (!other_pair->second->operator==(*pair.second)) return false;
for (size_t i = 0; i < rules.size(); i++) {
auto &pair = rules[i];
auto &other_pair = other.rules[i];
if (other_pair.first != pair.first) return false;
if (!other_pair.second->operator==(*pair.second)) return false;
}
return true;
}
string Grammar::start_rule_name() const {
return rules.front().first;
}
ostream& operator<<(ostream &stream, const Grammar &grammar) {
stream << string("#<grammar");

17
src/compiler/prepare_grammar/expand_repeats.cc
View file

@ -1,5 +1,5 @@
#include "compiler/prepare_grammar/expand_repeats.h"
#include <map>
#include <vector>
#include <string>
#include "compiler/prepared_grammar.h"
#include "compiler/rules/visitor.h"
@ -11,8 +11,9 @@
namespace tree_sitter {
using std::string;
using std::vector;
using std::pair;
using std::to_string;
using std::map;
using std::make_shared;
using rules::rule_ptr;
using rules::Blank;
@ -33,7 +34,7 @@ namespace tree_sitter {
void visit(const Repeat *rule) {
rule_ptr inner_rule = apply(rule->content);
string helper_rule_name = string("repeat_helper") + to_string(aux_rules.size() + 1);
aux_rules.insert({ helper_rule_name, make_repeat_helper(helper_rule_name, inner_rule) });
aux_rules.push_back({ helper_rule_name, make_repeat_helper(helper_rule_name, inner_rule) });
value = make_shared<Symbol>(helper_rule_name, rules::SymbolTypeAuxiliary);
}
@ -50,19 +51,19 @@ namespace tree_sitter {
}
public:
map<const string, const rule_ptr> aux_rules;
vector<pair<string, const rules::rule_ptr>> aux_rules;
};
PreparedGrammar expand_repeats(const PreparedGrammar &grammar) {
map<const string, const rule_ptr> rules, aux_rules(grammar.aux_rules);
vector<pair<string, rules::rule_ptr>> rules, aux_rules(grammar.aux_rules);
ExpandRepeats expander;
for (auto &pair : grammar.rules)
rules.insert({ pair.first, expander.apply(pair.second) });
rules.push_back({ pair.first, expander.apply(pair.second) });
aux_rules.insert(expander.aux_rules.begin(), expander.aux_rules.end());
aux_rules.insert(aux_rules.end(), expander.aux_rules.begin(), expander.aux_rules.end());
return PreparedGrammar(grammar.start_rule_name, rules, aux_rules);
return PreparedGrammar(rules, aux_rules);
}
}
}

24
src/compiler/prepare_grammar/extract_tokens.cc
View file

@ -1,5 +1,5 @@
#include "compiler/prepare_grammar/extract_tokens.h"
#include <map>
#include <vector>
#include <string>
#include "tree_sitter/compiler.h"
#include "compiler/prepared_grammar.h"
@ -16,7 +16,7 @@ namespace tree_sitter {
using std::pair;
using std::string;
using std::to_string;
using std::map;
using std::vector;
using std::make_shared;
using rules::rule_ptr;
@ -41,7 +41,7 @@ namespace tree_sitter {
if (*pair.second == *rule)
return pair.first;
string name = "token" + to_string(tokens.size() + 1);
tokens.insert({ name, rule });
tokens.push_back({ name, rule });
return name;
}
@ -67,36 +67,36 @@ namespace tree_sitter {
}
public:
map<const string, const rule_ptr> tokens;
vector<pair<string, const rules::rule_ptr>> tokens;
};
pair<PreparedGrammar, PreparedGrammar> extract_tokens(const PreparedGrammar &input_grammar) {
map<const string, const rule_ptr> rules, tokens, aux_rules, aux_tokens;
vector<pair<string, rules::rule_ptr>> rules, tokens, aux_rules, aux_tokens;
TokenExtractor extractor;
for (auto &pair : input_grammar.rules) {
string name = pair.first;
rule_ptr rule = pair.second;
if (IsToken().apply(rule))
tokens.insert({ name, rule });
tokens.push_back({ name, rule });
else
rules.insert({ name, extractor.apply(rule) });
rules.push_back({ name, extractor.apply(rule) });
}
for (auto &pair : input_grammar.aux_rules) {
string name = pair.first;
rule_ptr rule = pair.second;
if (IsToken().apply(rule))
aux_tokens.insert({ name, rule });
aux_tokens.push_back({ name, rule });
else
aux_rules.insert({ name, extractor.apply(rule) });
aux_rules.push_back({ name, extractor.apply(rule) });
}
aux_tokens.insert(extractor.tokens.begin(), extractor.tokens.end());
aux_tokens.insert(aux_tokens.end(), extractor.tokens.begin(), extractor.tokens.end());
return {
PreparedGrammar(input_grammar.start_rule_name, rules, aux_rules),
PreparedGrammar("", tokens, aux_tokens)
PreparedGrammar(rules, aux_rules),
PreparedGrammar(tokens, aux_tokens)
};
}
}

37
src/compiler/prepared_grammar.cc
View file

@ -1,28 +1,19 @@
#include "compiler/prepared_grammar.h"
#include <map>
#include <vector>
#include <string>
#include <utility>
#include "compiler/rules/symbol.h"
namespace tree_sitter {
using std::string;
using std::initializer_list;
using std::pair;
using std::map;
using std::ostream;
using rules::rule_ptr;
using rules::Symbol;
PreparedGrammar::PreparedGrammar(std::string start_rule_name,
const map<const string, const rule_ptr> &rules,
const map<const string, const rule_ptr> &aux_rules) :
Grammar(start_rule_name, rules),
aux_rules(aux_rules) {}
PreparedGrammar::PreparedGrammar(std::string start_rule_name,
const initializer_list<pair<const string, const rule_ptr>> &rules,
const initializer_list<pair<const string, const rule_ptr>> &aux_rules) :
Grammar(start_rule_name, rules),
PreparedGrammar::PreparedGrammar(const std::vector<std::pair<std::string, rules::rule_ptr>> &rules,
const std::vector<std::pair<std::string, rules::rule_ptr>> &aux_rules) :
Grammar(rules),
aux_rules(aux_rules) {}
PreparedGrammar::PreparedGrammar(const Grammar &grammar) :
@ -30,21 +21,21 @@ namespace tree_sitter {
aux_rules({}) {}
const rule_ptr PreparedGrammar::rule(const Symbol &symbol) const {
auto map = symbol.is_auxiliary() ? aux_rules : rules;
auto iter = map.find(symbol.name);
if (iter != map.end())
return iter->second;
else
return rule_ptr();
auto rule_set = symbol.is_auxiliary() ? aux_rules : rules;
for (auto &pair : rule_set)
if (pair.first == symbol.name)
return pair.second;
return rule_ptr();
}
bool PreparedGrammar::operator==(const PreparedGrammar &other) const {
if (!Grammar::operator==(other)) return false;
if (other.aux_rules.size() != aux_rules.size()) return false;
for (auto pair : aux_rules) {
auto other_pair = other.aux_rules.find(pair.first);
if (other_pair == other.aux_rules.end()) return false;
if (!other_pair->second->operator==(*pair.second)) return false;
for (size_t i = 0; i < aux_rules.size(); i++) {
auto &pair = aux_rules[i];
auto &other_pair = other.aux_rules[i];
if (other_pair.first != pair.first) return false;
if (!other_pair.second->operator==(*pair.second)) return false;
}
return true;
}

12
src/compiler/prepared_grammar.h
View file

@ -1,7 +1,7 @@
#ifndef COMPILER_PREPARED_GRAMMAR_H_
#define COMPILER_PREPARED_GRAMMAR_H_
#include <map>
#include <vector>
#include <string>
#include <utility>
#include "tree_sitter/compiler.h"
@ -10,19 +10,15 @@
namespace tree_sitter {
class PreparedGrammar : public Grammar {
public:
PreparedGrammar(std::string start_rule_name,
const std::map<const std::string, const rules::rule_ptr> &rules,
const std::map<const std::string, const rules::rule_ptr> &aux_rules);
PreparedGrammar(std::string start_rule_name,
const std::initializer_list<std::pair<const std::string, const rules::rule_ptr>> &rules,
const std::initializer_list<std::pair<const std::string, const rules::rule_ptr>> &aux_rules);
PreparedGrammar(const std::vector<std::pair<std::string, rules::rule_ptr>> &rules,
const std::vector<std::pair<std::string, rules::rule_ptr>> &aux_rules);
PreparedGrammar(const Grammar &grammar);
bool operator==(const PreparedGrammar &other) const;
bool has_definition(const rules::Symbol &symbol) const;
const rules::rule_ptr rule(const rules::Symbol &symbol) const;
const std::map<const std::string, const rules::rule_ptr> aux_rules;
const std::vector<std::pair<std::string, rules::rule_ptr>> aux_rules;
};
std::ostream& operator<<(std::ostream &stream, const PreparedGrammar &grammar);