traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove subclasses of Symbol for terminals and non-terminals

Browse source
This commit is contained in:
Max Brunsfeld 2014-01-04 15:01:06 -08:00
parent ed80d9cf52
commit 29c81167c0
23 changed files with 88 additions and 191 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/compiler/grammar/extract_tokens.cpp
View file

@ -19,7 +19,7 @@ namespace tree_sitter {
return value;
} else {
string token_name = add_token(rule);
return rules::token(token_name);
return rules::sym(token_name);
}
}

4
src/compiler/grammar/grammar.cpp
View file

@ -38,6 +38,10 @@ namespace tree_sitter {
return true;
}
bool Grammar::has_definition(const rules::Symbol &symbol) const {
return rules.find(symbol.name) != rules.end();
}
std::ostream& operator<<(std::ostream &stream, const Grammar &grammar) {
stream << string("#<grammar: ");
bool started = false;

1
src/compiler/grammar/grammar.h
View file

@ -15,6 +15,7 @@ namespace tree_sitter {
const std::string start_rule_name;
std::vector<std::string> rule_names() const;
bool operator==(const Grammar &other) const;
bool has_definition(const rules::Symbol &symbol) const;
const std::unordered_map<std::string, const rules::rule_ptr> rules;
};

18
src/compiler/lr/item.cpp
View file

@ -28,10 +28,10 @@ namespace tree_sitter {
});
};
vector<rules::NonTerminal> Item::next_symbols() const {
vector<rules::NonTerminal> result;
vector<rules::Symbol> Item::next_symbols() const {
vector<rules::Symbol> result;
for (auto pair : lr::transitions(rule)) {
auto sym = dynamic_pointer_cast<const rules::NonTerminal>(pair.first);
auto sym = dynamic_pointer_cast<const rules::Symbol>(pair.first);
if (sym) result.push_back(*sym);
}
return result;
@ -40,24 +40,24 @@ namespace tree_sitter {
bool Item::operator==(const Item &other) const {
bool rule_names_eq = other.rule_name == rule_name;
bool rules_eq = (*other.rule == *rule);
return rule_names_eq && rules_eq;
bool consumed_sym_counts_eq = (other.consumed_sym_count == consumed_sym_count);
return rule_names_eq && rules_eq && consumed_sym_counts_eq;
}
bool Item::is_done() const {
for (auto pair : transitions()) {
if (*pair.first == rules::Blank()) return true;
}
for (auto pair : transitions())
if (*pair.first == rules::Blank())
return true;
return false;
}
std::ostream& operator<<(ostream &stream, const Item &item) {
stream <<
return stream <<
string("#<item '") <<
item.rule_name <<
string("' ") <<
*item.rule <<
string(">");
return stream;
}
}
}

4
src/compiler/lr/item.h
View file

@ -3,7 +3,7 @@
#include <string>
#include "rule.h"
#include "non_terminal.h"
#include "symbol.h"
#include "transition_map.h"
namespace tree_sitter {
@ -20,7 +20,7 @@ namespace tree_sitter {
static Item at_beginning_of_token(const std::string &rule_name, const Grammar &grammar);
transition_map<rules::Rule, Item> transitions() const;
std::vector<rules::NonTerminal> next_symbols() const;
std::vector<rules::Symbol> next_symbols() const;
bool operator==(const Item &other) const;
bool is_done() const;

51
src/compiler/lr/item_set.cpp
View file

@ -1,10 +1,12 @@
#include "item_set.h"
using std::vector;
using std::set;
using std::initializer_list;
using std::dynamic_pointer_cast;
using std::ostream;
using std::string;
using std::make_shared;
namespace tree_sitter {
namespace lr {
@ -18,9 +20,11 @@ namespace tree_sitter {
static void add_item(vector<Item> &vector, const Item &item, const Grammar &grammar) {
if (!vector_contains(vector, item)) {
vector.push_back(item);
for (rules::NonTerminal rule : item.next_symbols()) {
Item next_item = Item::at_beginning_of_rule(rule.name, grammar);
add_item(vector, next_item, grammar);
for (rules::Symbol rule : item.next_symbols()) {
if (grammar.has_definition(rule)) {
Item next_item = Item::at_beginning_of_rule(rule.name, grammar);
add_item(vector, next_item, grammar);
}
}
}
}
@ -33,17 +37,35 @@ namespace tree_sitter {
ItemSet::ItemSet(const Item &item, const Grammar &grammar) : contents(closure_in_grammar(item, grammar)) {}
transition_map<rules::Rule, ItemSet> ItemSet::all_transitions(const Grammar &grammar) const {
transition_map<rules::Rule, ItemSet> result;
for (auto item : *this) {
auto item_transitions = item.transitions();
for (auto pair : item_transitions) {
result.add(pair.first, std::make_shared<ItemSet>(*pair.second, grammar));
template<typename RuleClass>
transition_map<RuleClass, ItemSet> transitions(const ItemSet *item_set, const Grammar &grammar) {
transition_map<RuleClass, ItemSet> result;
for (Item item : *item_set) {
for (auto transition : item.transitions()) {
auto rule = dynamic_pointer_cast<const RuleClass>(transition.first);
if (rule.get()) result.add(rule, make_shared<ItemSet>(*transition.second, grammar));
}
}
return result;
}
transition_map<rules::Character, ItemSet> ItemSet::char_transitions(const Grammar &grammar) const {
return transitions<rules::Character>(this, grammar);
}
transition_map<rules::Symbol, ItemSet> ItemSet::sym_transitions(const Grammar &grammar) const {
return transitions<rules::Symbol>(this, grammar);
}
set<rules::Symbol> ItemSet::next_terminal_symbols(const Grammar &grammar) const {
set<rules::Symbol> result;
for (Item item : *this)
for (rules::Symbol symbol : item.next_symbols())
if (!grammar.has_definition(symbol))
result.insert(symbol);
return result;
}
bool ItemSet::operator==(const tree_sitter::lr::ItemSet &other) const {
return contents == other.contents;
}
@ -66,12 +88,9 @@ namespace tree_sitter {
ostream& operator<<(ostream &stream, const ItemSet &item_set) {
stream << string("#<item_set ");
for (Item item : item_set) {
stream << item;
stream << string(" ");
}
stream << string(">");
return stream;
for (Item item : item_set)
stream << item << string(" ");
return stream << string(">");
}
}
}

23
src/compiler/lr/item_set.h
View file

@ -3,6 +3,7 @@
#include "item.h"
#include "grammar.h"
#include <set>
namespace tree_sitter {
namespace lr {
@ -22,26 +23,10 @@ namespace tree_sitter {
const_iterator end() const;
size_t size() const;
transition_map<rules::Rule, ItemSet> all_transitions(const Grammar &grammar) const;
template<typename RuleClass>
transition_map<RuleClass, ItemSet> transitions(const Grammar &grammar) const {
transition_map<RuleClass, ItemSet> result;
for (auto transition : all_transitions(grammar)) {
auto rule = std::dynamic_pointer_cast<const RuleClass>(transition.first);
if (rule.get()) result.add(rule, transition.second);
}
return result;
}
template<typename RuleClass>
std::vector<RuleClass> next_inputs(const Grammar &grammar) const {
std::vector<RuleClass> result;
for (auto pair : transitions<RuleClass>(grammar))
result.push_back(*pair.first);
return result;
}
transition_map<rules::Character, ItemSet> char_transitions(const Grammar &grammar) const;
transition_map<rules::Symbol, ItemSet> sym_transitions(const Grammar &grammar) const;
std::set<rules::Symbol> next_terminal_symbols(const Grammar &grammar) const;
bool operator==(const ItemSet &other) const;
};

10
src/compiler/lr/table_builder.cpp
View file

@ -32,7 +32,9 @@ namespace tree_sitter {
}
void add_shift_actions(const ItemSet &item_set, size_t state_index) {
for (auto transition : item_set.transitions<rules::Symbol>(grammar)) {
auto x = item_set.sym_transitions(grammar);
for (auto transition : x) {
rules::Symbol symbol = *transition.first;
ItemSet item_set = *transition.second;
size_t new_state_index = add_parse_state(item_set);
@ -41,7 +43,7 @@ namespace tree_sitter {
}
void add_advance_actions(const ItemSet &item_set, size_t state_index) {
for (auto transition : item_set.transitions<rules::Character>(grammar)) {
for (auto transition : item_set.char_transitions(grammar)) {
rules::Character rule = *transition.first;
ItemSet item_set = *transition.second;
size_t new_state_index = add_lex_state(item_set);
@ -82,8 +84,8 @@ namespace tree_sitter {
ItemSet lex_item_set_for_parse_item_set(const ItemSet &parse_item_set) {
vector<Item> items;
for (rules::Token token : parse_item_set.next_inputs<rules::Token>(grammar))
items.push_back(Item::at_beginning_of_token(token.name, lex_grammar));
for (rules::Symbol symbol : parse_item_set.next_terminal_symbols(grammar))
items.push_back(Item::at_beginning_of_token(symbol.name, lex_grammar));
return ItemSet(items);
}

4
src/compiler/lr/transitions.cpp
View file

@ -21,10 +21,6 @@ namespace tree_sitter {
value = transition_map<Rule, Rule>({{ rule->copy(), blank() }});
}
void visit(const Token *rule) {
value = transition_map<Rule, Rule>({{ rule->copy(), blank() }});
}
void visit(const Choice *rule) {
value = transitions(rule->left);
value.merge(transitions(rule->right), [&](rule_ptr left, rule_ptr right) -> rule_ptr {

28
src/compiler/rules/non_terminal.cpp
View file

@ -1,28 +0,0 @@
#include "rules.h"
#include "transition_map.h"
using std::string;
using std::hash;
namespace tree_sitter {
namespace rules {
NonTerminal::NonTerminal(const std::string &name) : Symbol(name) {};
bool NonTerminal::operator==(const Rule &rule) const {
const NonTerminal *other = dynamic_cast<const NonTerminal *>(&rule);
return other && (other->name == name);
}
rule_ptr NonTerminal::copy() const {
return std::make_shared<NonTerminal>(*this);
}
string NonTerminal::to_string() const {
return string("#<non-terminal '") + name + "'>";
}
void NonTerminal::accept(Visitor &visitor) const {
visitor.visit(this);
}
}
}

20
src/compiler/rules/non_terminal.h
View file

@ -1,20 +0,0 @@
#ifndef __tree_sitter__non_terminal__
#define __tree_sitter__non_terminal__
#include "symbol.h"
namespace tree_sitter {
namespace rules {
class NonTerminal : public Symbol {
public:
NonTerminal(const std::string &name);
bool operator==(const Rule& other) const;
rule_ptr copy() const;
std::string to_string() const;
void accept(Visitor &visitor) const;
};
}
}
#endif

6
src/compiler/rules/rules.cpp
View file

@ -47,11 +47,7 @@ namespace tree_sitter {
}
sym_ptr sym(const string &name) {
return make_shared<NonTerminal>(name);
}
rule_ptr token(const std::string &name) {
return make_shared<Token>(name);
return make_shared<Symbol>(name);
}
}
}

3
src/compiler/rules/rules.h
View file

@ -4,14 +4,12 @@
#include "rule.h"
#include "blank.h"
#include "symbol.h"
#include "token.h"
#include "choice.h"
#include "seq.h"
#include "string.h"
#include "pattern.h"
#include "character.h"
#include "repeat.h"
#include "non_terminal.h"
#include "visitor.h"
namespace tree_sitter {
@ -26,7 +24,6 @@ namespace tree_sitter {
rule_ptr seq(const std::initializer_list<rule_ptr> &rules);
rule_ptr str(const std::string &value);
sym_ptr sym(const std::string &name);
rule_ptr token(const std::string &name);
}
}

4
src/compiler/rules/symbol.cpp
View file

@ -25,6 +25,10 @@ namespace tree_sitter {
return string("#<sym '") + name + "'>";
}
bool Symbol::operator<(const Symbol &other) const {
return name < other.name;
}
void Symbol::accept(Visitor &visitor) const {
visitor.visit(this);
}

7
src/compiler/rules/symbol.h
View file

@ -14,6 +14,7 @@ namespace tree_sitter {
rule_ptr copy() const;
std::string to_string() const;
void accept(Visitor &visitor) const;
bool operator<(const Symbol &other) const;
const std::string name;
};
@ -22,4 +23,10 @@ namespace tree_sitter {
}
}
namespace std {
template<>
struct hash<tree_sitter::rules::Symbol> : hash<tree_sitter::rules::Rule> {};
}
#endif

28
src/compiler/rules/token.cpp
View file

@ -1,28 +0,0 @@
#include "rules.h"
#include "transition_map.h"
using std::string;
using std::hash;
namespace tree_sitter {
namespace rules {
Token::Token(const std::string &name) : Symbol(name) {};
bool Token::operator==(const Rule &rule) const {
const Token *other = dynamic_cast<const Token *>(&rule);
return other && (other->name == name);
}
rule_ptr Token::copy() const {
return std::make_shared<Token>(*this);
}
string Token::to_string() const {
return string("#<token '") + name + "'>";
}
void Token::accept(Visitor &visitor) const {
visitor.visit(this);
}
}
}

20
src/compiler/rules/token.h
View file

@ -1,20 +0,0 @@
#ifndef __tree_sitter__token__
#define __tree_sitter__token__
#include "symbol.h"
namespace tree_sitter {
namespace rules {
class Token : public Symbol {
public:
Token(const std::string &name);
bool operator==(const Rule& other) const;
rule_ptr copy() const;
std::string to_string() const;
void accept(Visitor &visitor) const;
};
}
}
#endif

1
src/compiler/rules/visitor.cpp
View file

@ -11,6 +11,5 @@ namespace tree_sitter {
void Visitor::visit(const Seq *rule) { default_visit(rule); }
void Visitor::visit(const String *rule) { default_visit(rule); }
void Visitor::visit(const Pattern *rule) { default_visit(rule); }
void Visitor::visit(const Token *rule) { default_visit(rule); }
}
}

1
src/compiler/rules/visitor.h
View file

@ -16,7 +16,6 @@ namespace tree_sitter {
virtual void visit(const Seq *rule);
virtual void visit(const String *rule);
virtual void visit(const Pattern *rule);
virtual void visit(const Token *rule);
};
}
}