traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Store choice rules using vectors, not pairs

Browse source
This commit is contained in:
Max Brunsfeld 2014-04-26 19:09:22 -07:00
parent c9e0d4bdf0
commit 29bbff655c
9 changed files with 9273 additions and 8919 deletions
Download patch file Download diff file Expand all files Collapse all files

18072
examples/parsers/javascript.c
View file

File diff suppressed because it is too large Load diff

24
spec/compiler/rules/rules_spec.cc
View file

@ -1,24 +0,0 @@
#include "compiler_spec_helper.h"
#include "compiler/rules/rule.h"
using namespace rules;
START_TEST
describe("constructing rules", []() {
rule_ptr symbol1 = sym("1");
rule_ptr symbol2 = sym("2");
rule_ptr symbol3 = sym("3");
it("constructs binary trees", [&]() {
AssertThat(
seq({ symbol1, symbol2, symbol3 }),
EqualsPointer(seq({ seq({ symbol1, symbol2 }), symbol3 })));
AssertThat(
choice({ symbol1, symbol2, symbol3 }),
EqualsPointer(choice({ choice({ symbol1, symbol2 }), symbol3 })));
});
});
END_TEST

37
src/compiler/build_tables/first_set.cc
View file

@ -13,12 +13,6 @@ namespace tree_sitter {
using rules::ISymbol;
namespace build_tables {
set<ISymbol> set_union(const set<ISymbol> &left, const set<ISymbol> &right) {
set<ISymbol> result = left;
result.insert(right.begin(), right.end());
return result;
}
class FirstSet : public rules::RuleFn<set<ISymbol>> {
const PreparedGrammar *grammar;
set<ISymbol> visited_symbols;
@ -26,14 +20,11 @@ namespace tree_sitter {
explicit FirstSet(const PreparedGrammar *grammar) : grammar(grammar) {}
set<ISymbol> apply_to(const ISymbol *rule) {
if (visited_symbols.find(*rule) == visited_symbols.end()) {
visited_symbols.insert(*rule);
if (rule->is_token()) {
return set<ISymbol>({ *rule });
} else {
return apply(grammar->rule(*rule));
}
auto insertion_result = visited_symbols.insert(*rule);
if (insertion_result.second) {
return (rule->is_token()) ?
set<ISymbol>({ *rule }) :
apply(grammar->rule(*rule));
} else {
return set<ISymbol>();
}
@ -44,16 +35,21 @@ namespace tree_sitter {
}
set<ISymbol> apply_to(const rules::Choice *rule) {
return set_union(apply(rule->left), apply(rule->right));
set<ISymbol> result;
for (const auto &el : rule->elements) {
auto &&next_syms = apply(el);
result.insert(next_syms.begin(), next_syms.end());
}
return result;
}
set<ISymbol> apply_to(const rules::Seq *rule) {
auto result = apply(rule->left);
auto &&result = apply(rule->left);
if (rule_can_be_blank(rule->left, *grammar)) {
return set_union(result, apply(rule->right));
} else {
return result;
auto &&right_symbols = apply(rule->right);
result.insert(right_symbols.begin(), right_symbols.end());
}
return result;
}
};
@ -64,7 +60,8 @@ namespace tree_sitter {
set<ISymbol> first_set(const ParseItemSet &item_set, const PreparedGrammar &grammar) {
set<ISymbol> result;
for (auto &item : item_set) {
result = set_union(result, first_set(item.rule, grammar));
auto &&rule_set = first_set(item.rule, grammar);
result.insert(rule_set.begin(), rule_set.end());
if (rule_can_be_blank(item.rule, grammar))
result.insert(item.lookahead_sym);
}

4
src/compiler/build_tables/rule_can_be_blank.cc
View file

@ -24,7 +24,9 @@ namespace tree_sitter {
}
bool apply_to(const rules::Choice *rule) {
return apply(rule->left) || apply(rule->right);
for (const auto &element : rule->elements)
if (apply(element)) return true;
return false;
}
bool apply_to(const rules::Seq *rule) {

5
src/compiler/build_tables/rule_transitions.cc
View file

@ -62,8 +62,9 @@ namespace tree_sitter {
}
map<T, rule_ptr> apply_to(const rules::Choice *rule) {
auto result = this->apply(rule->left);
merge_transitions<T>(result, this->apply(rule->right));
map<T, rule_ptr> result;
for (const auto &el : rule->elements)
merge_transitions<T>(result, this->apply(el));
return result;
}

29
src/compiler/rules/choice.cc
View file

@ -1,29 +1,37 @@
#include "compiler/rules/choice.h"
#include <string>
#include <set>
#include "compiler/rules/visitor.h"
namespace tree_sitter {
using std::string;
using std::make_shared;
using std::vector;
using std::set;
using std::dynamic_pointer_cast;
namespace rules {
Choice::Choice(rule_ptr left, rule_ptr right) : left(left), right(right) {}
Choice::Choice(const vector<rule_ptr> &elements) : elements(elements) {}
rule_ptr Choice::Build(const vector<rule_ptr> &rules) {
rule_ptr result;
for (auto rule : rules)
result = result.get() ? make_shared<Choice>(result, rule) : rule;
return result;
rule_ptr Choice::Build(const vector<rule_ptr> &elements) {
return make_shared<Choice>(elements);
}
bool Choice::operator==(const Rule &rule) const {
const Choice *other = dynamic_cast<const Choice *>(&rule);
return other && (*other->left == *left) && (*other->right == *right);
if (!other) return false;
size_t size = elements.size();
if (size != other->elements.size()) return false;
for (size_t i = 0; i < size; i++)
if (!elements[i]->operator==(*other->elements[i])) return false;
return true;
}
size_t Choice::hash_code() const {
return left->hash_code() ^ right->hash_code();
size_t result = std::hash<size_t>()(elements.size());
for (const auto &element : elements)
result ^= element->hash_code();
return result;
}
rule_ptr Choice::copy() const {
@ -31,7 +39,10 @@ namespace tree_sitter {
}
string Choice::to_string() const {
return string("#<choice ") + left->to_string() + " " + right->to_string() + ">";
string result = "#<choice";
for (const auto &element : elements)
result += " " + element->to_string();
return result + ">";
}
void Choice::accept(Visitor *visitor) const {

5
src/compiler/rules/choice.h
View file

@ -9,7 +9,7 @@ namespace tree_sitter {
namespace rules {
class Choice : public Rule {
public:
Choice(rule_ptr left, rule_ptr right);
Choice(const std::vector<rule_ptr> &elements);
static rule_ptr Build(const std::vector<rule_ptr> &rules);
bool operator==(const Rule& other) const;
@ -18,8 +18,7 @@ namespace tree_sitter {
std::string to_string() const;
void accept(Visitor *visitor) const;
const rule_ptr left;
const rule_ptr right;
const std::vector<rule_ptr> elements;
};
}
}

9
src/compiler/rules/pattern.cc
View file

@ -15,6 +15,7 @@ namespace tree_sitter {
using std::hash;
using std::make_shared;
using std::set;
using std::vector;
class PatternParser {
public:
@ -24,12 +25,12 @@ namespace tree_sitter {
position(0) {}
rule_ptr rule() {
auto result = term();
vector<rule_ptr> choices = { term() };
while (has_more_input() && peek() == '|') {
next();
result = make_shared<Choice>(result, term());
choices.push_back(term());
}
return result;
return (choices.size() > 1) ? Choice::Build(choices) : choices.front();
}
private:
@ -54,7 +55,7 @@ namespace tree_sitter {
break;
case '?':
next();
result = make_shared<Choice>(result, make_shared<Blank>());
result = Choice::Build({ result, make_shared<Blank>() });
break;
}
}

7
src/compiler/rules/visitor.cc
View file

@ -11,13 +11,18 @@
#include "compiler/rules/repeat.h"
namespace tree_sitter {
using std::vector;
namespace rules {
rule_ptr IdentityRuleFn::default_apply(const Rule *rule) {
return rule->copy();
}
rule_ptr IdentityRuleFn::apply_to(const Choice *rule) {
return Choice::Build({ apply(rule->left), apply(rule->right) });
vector<rule_ptr> rules;
for (const auto &el : rule->elements)
rules.push_back(apply(el));
return Choice::Build(rules);
}
rule_ptr IdentityRuleFn::apply_to(const Seq *rule) {