traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Start work on building parse tables

Browse source
This commit is contained in:
Max Brunsfeld 2013-12-15 14:41:51 -08:00
parent 99c216f78a
commit 928f4f9deb
19 changed files with 419 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

14
TreeSitter.xcodeproj/project.pbxproj
View file

@ -18,10 +18,11 @@
12130622182C85D300FCF928 /* item_set.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 12130620182C85D300FCF928 /* item_set.cpp */; };
1214930E181E200B008E9BDA /* main.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 121492E9181E200B008E9BDA /* main.cpp */; };
1214930F181E200B008E9BDA /* rules_spec.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 121492EA181E200B008E9BDA /* rules_spec.cpp */; };
12512093182F307C00C9B56A /* parse_table_spec.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 12512092182F307C00C9B56A /* parse_table_spec.cpp */; };
12512093182F307C00C9B56A /* parse_table_builder_spec.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 12512092182F307C00C9B56A /* parse_table_builder_spec.cpp */; };
1251209B1830145300C9B56A /* rule.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1251209A1830145300C9B56A /* rule.cpp */; };
125120A018307DEC00C9B56A /* parse_table.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1251209E18307DEC00C9B56A /* parse_table.cpp */; };
125120A4183083BD00C9B56A /* arithmetic.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 125120A3183083BD00C9B56A /* arithmetic.cpp */; };
129D242C183EB1EB00FE9F71 /* parse_table_builder.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 129D242A183EB1EB00FE9F71 /* parse_table_builder.cpp */; };
12D1369D18328C5A005F3369 /* item_spec.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 12D1369C18328C5A005F3369 /* item_spec.cpp */; };
12D136A1183570F5005F3369 /* pattern_spec.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 12D136A0183570F5005F3369 /* pattern_spec.cpp */; };
12D136A4183678A2005F3369 /* repeat.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 12D136A2183678A2005F3369 /* repeat.cpp */; };
@ -135,12 +136,14 @@
121492C6181E200B008E9BDA /* igloo_framework.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = igloo_framework.h; sourceTree = "<group>"; };
121492E9181E200B008E9BDA /* main.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = main.cpp; path = spec/main.cpp; sourceTree = SOURCE_ROOT; };
121492EA181E200B008E9BDA /* rules_spec.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = rules_spec.cpp; path = spec/rules/rules_spec.cpp; sourceTree = SOURCE_ROOT; };
12512092182F307C00C9B56A /* parse_table_spec.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = parse_table_spec.cpp; path = spec/lr/parse_table_spec.cpp; sourceTree = SOURCE_ROOT; };
12512092182F307C00C9B56A /* parse_table_builder_spec.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = parse_table_builder_spec.cpp; path = spec/lr/parse_table_builder_spec.cpp; sourceTree = SOURCE_ROOT; };
1251209A1830145300C9B56A /* rule.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = rule.cpp; sourceTree = "<group>"; };
1251209E18307DEC00C9B56A /* parse_table.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = parse_table.cpp; sourceTree = "<group>"; };
1251209F18307DEC00C9B56A /* parse_table.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = parse_table.h; sourceTree = "<group>"; };
125120A218307FFD00C9B56A /* arithmetic.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = arithmetic.h; path = spec/test_grammars/arithmetic.h; sourceTree = SOURCE_ROOT; };
125120A3183083BD00C9B56A /* arithmetic.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = arithmetic.cpp; path = spec/test_grammars/arithmetic.cpp; sourceTree = SOURCE_ROOT; };
129D242A183EB1EB00FE9F71 /* parse_table_builder.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = parse_table_builder.cpp; sourceTree = "<group>"; };
129D242B183EB1EB00FE9F71 /* parse_table_builder.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = parse_table_builder.h; sourceTree = "<group>"; };
12C344421822F27700B07BE3 /* transition_map.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = transition_map.h; sourceTree = "<group>"; };
12D1369C18328C5A005F3369 /* item_spec.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = item_spec.cpp; path = spec/lr/item_spec.cpp; sourceTree = SOURCE_ROOT; };
12D1369E18342088005F3369 /* todo.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = todo.md; sourceTree = "<group>"; };
@ -206,6 +209,8 @@
12130621182C85D300FCF928 /* item_set.h */,
1251209E18307DEC00C9B56A /* parse_table.cpp */,
1251209F18307DEC00C9B56A /* parse_table.h */,
129D242A183EB1EB00FE9F71 /* parse_table_builder.cpp */,
129D242B183EB1EB00FE9F71 /* parse_table_builder.h */,
);
path = lr;
sourceTree = "<group>";
@ -214,7 +219,7 @@
isa = PBXGroup;
children = (
1213061D182C857100FCF928 /* item_set_spec.cpp */,
12512092182F307C00C9B56A /* parse_table_spec.cpp */,
12512092182F307C00C9B56A /* parse_table_builder_spec.cpp */,
12D1369C18328C5A005F3369 /* item_spec.cpp */,
);
name = lr;
@ -537,6 +542,7 @@
buildActionMask = 2147483647;
files = (
12130614182C3A1700FCF928 /* seq.cpp in Sources */,
129D242C183EB1EB00FE9F71 /* parse_table_builder.cpp in Sources */,
12D136A1183570F5005F3369 /* pattern_spec.cpp in Sources */,
125120A4183083BD00C9B56A /* arithmetic.cpp in Sources */,
1214930F181E200B008E9BDA /* rules_spec.cpp in Sources */,
@ -550,7 +556,7 @@
125120A018307DEC00C9B56A /* parse_table.cpp in Sources */,
1214930E181E200B008E9BDA /* main.cpp in Sources */,
12F9A651182DD6BC00FAF50C /* grammar.cpp in Sources */,
12512093182F307C00C9B56A /* parse_table_spec.cpp in Sources */,
12512093182F307C00C9B56A /* parse_table_builder_spec.cpp in Sources */,
1213061F182C857100FCF928 /* item_set_spec.cpp in Sources */,
12D136A4183678A2005F3369 /* repeat.cpp in Sources */,
12130622182C85D300FCF928 /* item_set.cpp in Sources */,

11
spec/lr/item_set_spec.cpp
View file

@ -9,7 +9,7 @@ static item_set_ptr item_set(const std::initializer_list<Item> &items) {
return item_set_ptr(new ItemSet(items));
}
Describe_Only(item_sets) {
Describe(item_sets) {
Grammar grammar = test_grammars::arithmetic();
It(computes_the_closure_of_an_item_set_under_symbol_expansion) {
@ -53,4 +53,13 @@ Describe_Only(item_sets) {
{ character('('), item_set({ Item("left_paren", blank(), 1) }) }
}), TransitionMap<ItemSet>::elements_equal));
}
It(can_be_hashed) {
ItemSet set1 = ItemSet(Item::at_beginning_of_rule("factor", grammar), grammar);
ItemSet set2 = ItemSet(Item::at_beginning_of_rule("factor", grammar), grammar);
AssertThat(std::hash<ItemSet>()(set1), Equals(std::hash<ItemSet>()(set2)));
ItemSet set3 = ItemSet(Item::at_beginning_of_rule("term", grammar), grammar);
AssertThat(std::hash<ItemSet>()(set1), !Equals(std::hash<ItemSet>()(set3)));
}
};

44
spec/lr/parse_table_builder_spec.cpp Normal file
View file

@ -0,0 +1,44 @@
#include "spec_helper.h"
#include "../test_grammars/arithmetic.h"
#include "parse_table_builder.h"
#include "parse_table.h"
using namespace tree_sitter::lr;
typedef std::unordered_set<ParseAction> actions;
Describe(ParseTableBuilder_test) {
Grammar grammar = test_grammars::arithmetic();
ParseTable table = ParseTableBuilder::build_table(grammar);
It(has_the_right_starting_state) {
AssertThat(table.actions_for(0), Equals(unordered_map<string, actions>({
{ "expression", actions({ ParseAction::Shift(1) }) },
{ "term", actions({ ParseAction::Shift(2) }) },
{ "factor", actions({ ParseAction::Shift(5) }) },
{ "variable", actions({ ParseAction::Shift(8) }) },
{ "number", actions({ ParseAction::Shift(8) }) },
{ "left_paren", actions({ ParseAction::Shift(9) }) }
})));
}
It(accepts_when_the_start_symbol_is_reduced) {
AssertThat(table.actions_for(1), Equals(unordered_map<string, actions>({
{ ParseTable::END_OF_INPUT, actions({ ParseAction::Accept() }) }
})));
}
It(has_the_right_next_states) {
AssertThat(table.actions_for(2), Equals(unordered_map<string, actions>({
{ "plus", actions({ ParseAction::Shift(3) }) },
})));
AssertThat(table.actions_for(3), Equals(unordered_map<string, actions>({
{ "variable", actions({ ParseAction::Shift(8) }) },
{ "factor", actions({ ParseAction::Shift(5) }) },
{ "left_paren", actions({ ParseAction::Shift(9) }) },
{ "number", actions({ ParseAction::Shift(8) }) },
{ "term", actions({ ParseAction::Shift(4) }) },
})));
}
};

10
spec/lr/parse_table_spec.cpp
View file

@ -1,10 +0,0 @@
#include "spec_helper.h"
#include "../test_grammars/arithmetic.h"
using namespace tree_sitter::lr;
Describe(build_parse_tables) {
Describe(lexing_tables) {
};
};

23
spec/spec_helper.cpp
View file

@ -4,3 +4,26 @@
EqualsContainerConstraint<TransitionMap<rules::Rule>, rule_tmap_comparator> EqualsTransitionMap(const TransitionMap<rules::Rule> &expected) {
return EqualsContainer(expected, TransitionMap<rules::Rule>::elements_equal);
}
namespace tree_sitter {
namespace lr {
std::ostream& operator<<(std::ostream &stream, const unordered_map<string, unordered_set<ParseAction>> &map) {
stream << string("{");
bool started = false;
for (auto pair : map) {
if (started) stream << string(", ");
stream << string("{") << pair.first << string(", [");
bool started_set = false;
for (ParseAction action : pair.second) {
if (started_set) stream << ", ";
stream << action;
started_set = true;
}
stream << string("]}");
started = true;
}
stream << string("}");
return stream;
}
}
}

8
spec/spec_helper.h
View file

@ -8,6 +8,8 @@
#include "item.h"
#include "item_set.h"
#include "grammar.h"
#include "parse_table.h"
#include "parse_table_builder.h"
using namespace tree_sitter;
using namespace std;
@ -17,4 +19,10 @@ using namespace igloo;
typedef bool (* rule_tmap_comparator)(const std::pair<rules::rule_ptr, rules::rule_ptr> &, const std::pair<rules::rule_ptr, rules::rule_ptr> &);
EqualsContainerConstraint<TransitionMap<rules::Rule>, rule_tmap_comparator> EqualsTransitionMap(const TransitionMap<rules::Rule> &expected);
namespace tree_sitter {
namespace lr {
std::ostream& operator<<(std::ostream &stream, const unordered_map<string, unordered_set<ParseAction>> &map);
}
}
#endif

11
src/grammar.cpp
View file

@ -1,5 +1,7 @@
#include "grammar.h"
using namespace std;
namespace tree_sitter {
Grammar::Grammar(const rule_map_init_list &rules) :
rules(rules),
@ -11,4 +13,13 @@ namespace tree_sitter {
rules::rule_ptr(nullptr) :
iter->second;
}
vector<string> Grammar::rule_names() const {
vector<string> result;
for (auto pair : rules) {
result.push_back(pair.first);
}
return result;
}
}

6
src/grammar.h
View file

@ -2,20 +2,20 @@
#define __TreeSitter__grammar__
#include <unordered_map>
#include <vector>
#include "rules.h"
namespace tree_sitter {
class Grammar {
typedef std::unordered_map<std::string, rules::rule_ptr> rule_map;
typedef std::initializer_list<std::pair<const std::string, rules::rule_ptr>> rule_map_init_list;
const rule_map rules;
public:
Grammar(const rule_map_init_list &rules);
const rules::rule_ptr rule(const std::string &) const;
const std::string start_rule_name;
private:
const rule_map rules;
std::vector<std::string> rule_names() const;
};
}

7
src/lr/item.cpp
View file

@ -1,6 +1,8 @@
#include "item.h"
#include "grammar.h"
#include <iostream>
using namespace std;
namespace tree_sitter {
@ -35,6 +37,10 @@ namespace tree_sitter {
return rule_names_eq && rules_eq;
}
bool Item::is_done() const {
return *rule == rules::Blank();
}
std::ostream& operator<<(ostream &stream, const Item &item) {
stream <<
string("(item '") <<
@ -47,4 +53,3 @@ namespace tree_sitter {
}
}

13
src/lr/item.h
View file

@ -17,6 +17,7 @@ namespace tree_sitter {
TransitionMap<Item> transitions() const;
std::vector<rules::sym_ptr> next_symbols() const;
bool operator==(const Item &other) const;
bool is_done() const;
const std::string rule_name;
const rules::rule_ptr rule;
@ -25,8 +26,20 @@ namespace tree_sitter {
typedef std::shared_ptr<const Item> item_ptr;
std::ostream& operator<<(std::ostream &stream, const Item &item);
}
}
namespace std {
template<>
struct hash<tree_sitter::lr::Item> {
size_t operator()(const tree_sitter::lr::Item &item) {
return
hash<std::string>()(item.rule_name) ^
hash<tree_sitter::rules::Rule>()(*item.rule) ^
hash<int>()(item.consumed_sym_count);
}
};
}
#endif

14
src/lr/item_set.h
View file

@ -10,6 +10,7 @@ namespace tree_sitter {
typedef std::shared_ptr<const ItemSet> item_set_ptr;
class ItemSet {
const std::vector<Item> contents;
public:
ItemSet(const std::vector<Item> &items);
ItemSet(const std::initializer_list<Item> &items);
@ -25,7 +26,6 @@ namespace tree_sitter {
TransitionMap<ItemSet> char_transitions(const Grammar &grammar) const;
bool operator==(const ItemSet &other) const;
const std::vector<Item> contents;
};
typedef std::shared_ptr<const ItemSet> item_set_ptr;
@ -33,4 +33,16 @@ namespace tree_sitter {
}
}
namespace std {
template<>
struct hash<tree_sitter::lr::ItemSet> {
size_t operator()(const tree_sitter::lr::ItemSet &item_set) const {
size_t result = hash<size_t>()(item_set.size());
for (auto item : item_set)
result ^= hash<tree_sitter::lr::Item>()(item);
return result;
}
};
}
#endif

101
src/lr/parse_table.cpp
View file

@ -1 +1,102 @@
#include "parse_table.h"
using namespace std;
namespace tree_sitter {
namespace lr {
// Action
ParseAction::ParseAction() :
type(ParseActionTypeError),
state_index(-1),
symbol_name(""),
child_symbol_count(-1) {};
ParseAction::ParseAction(ParseActionType type, size_t state_index, string symbol_name, size_t child_symbol_count) :
type(type),
state_index(state_index),
symbol_name(symbol_name),
child_symbol_count(child_symbol_count) {};
ParseAction ParseAction::Error() {
return ParseAction(ParseActionTypeError, -1, "", -1);
}
ParseAction ParseAction::Accept() {
return ParseAction(ParseActionTypeAccept, -1, "", -1);
}
ParseAction ParseAction::Shift(size_t state_index) {
return ParseAction(ParseActionTypeShift, state_index, "", -1);
}
ParseAction ParseAction::Reduce(std::string symbol_name, size_t child_symbol_count) {
return ParseAction(ParseActionTypeReduce, -1, symbol_name, child_symbol_count);
}
bool ParseAction::operator==(const ParseAction &other) const {
bool types_eq = type == other.type;
bool state_indices_eq = state_index == other.state_index;
bool symbol_ids_eq = symbol_name == other.symbol_name;
bool child_symbol_counts_eq = child_symbol_count == other.child_symbol_count;
return types_eq && state_indices_eq && symbol_ids_eq && child_symbol_counts_eq;
}
ostream& operator<<(ostream &stream, const ParseAction &action) {
switch (action.type) {
case ParseActionTypeAccept:
return stream << string("accept");
case ParseActionTypeAdvance:
return stream << (string("(advance ") + to_string(action.state_index) + ")");
case ParseActionTypeShift:
return stream << (string("(shift ") + to_string(action.state_index) + ")");
case ParseActionTypeReduce:
return stream << (string("(reduce ") + action.symbol_name + ")");
case ParseActionTypeError:
return stream << string("error");
}
}
// State
ParseState::ParseState() : actions(unordered_map<string, unordered_set<ParseAction>>()) {}
// Table
unordered_map<string, size_t> get_symbol_id_map(const vector<string> &names) {
unordered_map<string, size_t> result;
size_t i = 0;
for (string name : names) {
result[name] = i;
i++;
}
return result;
}
ParseTable::ParseTable(vector<string> symbol_names) :
symbol_ids(get_symbol_id_map(symbol_names)),
symbol_names(symbol_names),
states(vector<ParseState>()) {};
ParseState ParseTable::get_state(size_t index) const {
return states[index];
}
size_t ParseTable::add_state() {
states.push_back(ParseState());
return states.size() - 1;
}
void ParseTable::add_action(size_t state_index, string sym_name, ParseAction action) {
states[state_index].actions[sym_name].insert(action);
}
ParseState ParseTable::starting_state() const {
return states[0];
}
unordered_map<string, unordered_set<ParseAction>> ParseTable::actions_for(size_t state_index) const {
return states[state_index].actions;
}
const string ParseTable::START = "__START__";
const string ParseTable::END_OF_INPUT = "__END__";
}
}

69
src/lr/parse_table.h
View file

@ -1,13 +1,78 @@
#ifndef __TreeSitter__parse_table__
#define __TreeSitter__parse_table__
#include <unordered_map>
#include <vector>
#include <unordered_set>
#include "rule.h"
namespace tree_sitter {
namespace lr {
class ParseTable {
typedef enum {
ParseActionTypeAccept,
ParseActionTypeAdvance,
ParseActionTypeError,
ParseActionTypeShift,
ParseActionTypeReduce,
} ParseActionType;
class ParseAction {
public:
ParseTable();
ParseAction();
ParseAction(ParseActionType type, size_t state_index, std::string symbol_name, size_t child_symbol_count);
bool operator==(const ParseAction &action) const;
static ParseAction Accept();
static ParseAction Advance(size_t state_index);
static ParseAction Error();
static ParseAction Shift(size_t state_index);
static ParseAction Reduce(std::string symbol_name, size_t child_symbol_count);
ParseActionType type;
size_t child_symbol_count;
std::string symbol_name;
size_t state_index;
};
std::ostream& operator<<(std::ostream &stream, const ParseAction &item);
class ParseState {
public:
ParseState();
std::unordered_map<std::string, std::unordered_set<ParseAction>> actions;
};
class ParseTable {
std::vector<ParseState> states;
public:
const std::unordered_map<std::string, size_t> symbol_ids;
const std::vector<std::string> symbol_names;
ParseTable(std::vector<std::string> rule_names);
ParseState starting_state() const;
ParseState get_state(size_t index) const;
ParseAction action_for(size_t state_index, std::string symbol_name) const;
std::unordered_map<std::string, std::unordered_set<ParseAction>> actions_for(size_t state_index) const;
size_t add_state();
void add_action(size_t state_index, std::string symbol_name, ParseAction action);
static const std::string START;
static const std::string END_OF_INPUT;
};
}
}
namespace std {
template<>
struct hash<tree_sitter::lr::ParseAction> {
size_t operator()(const tree_sitter::lr::ParseAction &action) const {
return (
hash<int>()(action.type) ^
hash<string>()(action.symbol_name) ^
hash<size_t>()(action.state_index) ^
hash<size_t>()(action.child_symbol_count));
}
};
}
#endif

59
src/lr/parse_table_builder.cpp Normal file
View file

@ -0,0 +1,59 @@
#include "parse_table_builder.h"
#include "item_set.h"
#include "rules.h"
using namespace std;
namespace tree_sitter {
namespace lr {
static int NOT_FOUND = -1;
ParseTable ParseTableBuilder::build_table(const tree_sitter::Grammar &grammar) {
auto builder = ParseTableBuilder(grammar);
builder.build();
return builder.table;
}
ParseTableBuilder::ParseTableBuilder(const Grammar &grammar) :
grammar(grammar),
table(ParseTable(grammar.rule_names())),
state_indices(unordered_map<size_t, size_t>()) {};
void ParseTableBuilder::build() {
auto item = Item(ParseTable::START, rules::sym(grammar.start_rule_name), 0);
auto item_set = std::make_shared<ItemSet>(item, grammar);
add_item_set(item_set);
}
size_t ParseTableBuilder::add_item_set(const shared_ptr<const ItemSet> item_set) {
auto state_index = state_index_for_item_set(*item_set);
if (state_index == NOT_FOUND) {
state_index = table.add_state();
state_indices[std::hash<ItemSet>()(*item_set)] = state_index;
for (auto transition : item_set->sym_transitions(grammar)) {
rules::sym_ptr symbol = static_pointer_cast<const rules::Symbol>(transition.first);
size_t new_state_index = add_item_set(transition.second);
table.add_action(state_index, symbol->name, ParseAction::Shift(new_state_index));
}
for (Item item : *item_set) {
if (item.is_done()) {
if (item.rule_name == ParseTable::START) {
table.add_action(state_index, ParseTable::END_OF_INPUT, ParseAction::Accept());
} else {
for (string rule_name : table.symbol_names)
table.add_action(state_index, rule_name, ParseAction::Reduce(item.rule_name, item.consumed_sym_count));
}
}
}
}
return state_index;
}
long ParseTableBuilder::state_index_for_item_set(const ItemSet &item_set) const {
auto entry = state_indices.find(std::hash<ItemSet>()(item_set));
return (entry == state_indices.end()) ? NOT_FOUND : entry->second;
}
}
}

34
src/lr/parse_table_builder.h Normal file
View file

@ -0,0 +1,34 @@
#ifndef __TreeSitter__parse_table_builder__
#define __TreeSitter__parse_table_builder__
#include <unordered_map>
#include "grammar.h"
#include "item_set.h"
#include "parse_table.h"
namespace tree_sitter {
namespace lr {
class ItemSet;
struct ItemSetPointerHasher {
std::size_t operator()(const std::shared_ptr<const ItemSet> &item_set) const {
return std::hash<ItemSet>()(*item_set);
}
};
class ParseTableBuilder {
const Grammar grammar;
ParseTable table;
std::unordered_map<size_t, size_t> state_indices;
size_t add_item_set(const std::shared_ptr<const ItemSet> item_set);
long state_index_for_item_set(const ItemSet &item_set) const;
public:
ParseTableBuilder(const Grammar &grammar);
static ParseTable build_table(const Grammar &grammar);
void build();
};
}
}
#endif

5
src/rules/rule.cpp
View file

@ -1,9 +1,10 @@
#include "rule.h"
#include <set>
namespace tree_sitter {
namespace rules {
bool Rule::operator==(const rule_ptr other) const {
return true;
size_t Rule::hash_code() const {
return std::hash<std::string>()(to_string());
}
std::ostream& operator<<(std::ostream& stream, const Rule &rule)

10
src/rules/rule.h
View file

@ -15,7 +15,7 @@ namespace tree_sitter {
virtual TransitionMap<Rule> transitions() const = 0;
virtual bool operator==(const Rule& other) const = 0;
virtual std::string to_string() const = 0;
bool operator==(const rule_ptr other) const;
virtual size_t hash_code() const;
};
@ -24,5 +24,13 @@ namespace tree_sitter {
}
}
namespace std {
template<>
struct hash<tree_sitter::rules::Rule> {
size_t operator()(const tree_sitter::rules::Rule &rule) {
return rule.hash_code();
}
};
}
#endif

7
src/transition_map.h
View file

@ -15,7 +15,6 @@ namespace tree_sitter {
typedef std::vector<pair_type> contents_type;
public:
static bool elements_equal(const pair_type &left, const pair_type &right) {
return (*left.first == *right.first) && (*left.second == *right.second);
}
@ -72,8 +71,10 @@ namespace tree_sitter {
template<typename NewMappedType>
TransitionMap<NewMappedType> map(std::function<std::shared_ptr<const NewMappedType>(mapped_ptr)> map_fn) {
TransitionMap<NewMappedType> result;
for (pair_type pair : *this)
result.add(pair.first, map_fn(pair.second));
for (pair_type pair : *this) {
auto new_value = map_fn(pair.second);
result.add(pair.first, new_value);
}
return result;
}

2
todo.md
View file

@ -19,7 +19,7 @@ Then generate a C function for a ParseTable
For a Parser, the ParseActions can be any of:
- Accept(symbol)
- Shift(symbol)
- Shift(state_index)
- Reduce(symbol, number of child symbols)
# normalize grammars