traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Generate parse tables and code using pre-processed grammars

Browse source
This commit is contained in:
Max Brunsfeld 2014-01-03 22:42:05 -08:00
parent 2621f06315
commit 2afd8843dc
27 changed files with 272 additions and 285 deletions
Download patch file Download diff file Expand all files Collapse all files

11
spec/compiler/generate_parsers.cpp
View file

@ -1,6 +1,7 @@
#include "spec_helper.h"
#include "table_builder.h"
#include "parse_table.h"
#include "prepare_grammar.h"
#include "c_code.h"
#include <fstream>
@ -11,9 +12,13 @@ describe("code generation", []() {
it("works for the arithmetic grammar", [&]() {
Grammar grammar = test_grammars::arithmetic();
auto tables = lr::build_tables(grammar);
string code = code_gen::c_code(grammar, tables.first, tables.second);
std::ofstream(test_parser_dir + "/arithmetic.c") << code;
auto grammars = prepare_grammar(grammar);
auto tables = lr::build_tables(grammars.first, grammars.second);
auto rule_names = grammars.first.rule_names();
auto token_names = grammars.second.rule_names();
rule_names.insert(rule_names.end(), token_names.begin(), token_names.end());
auto code = code_gen::c_code(rule_names, tables.first, tables.second);
ofstream(test_parser_dir + "/arithmetic.c") << code;
});
});

68
spec/compiler/lr/item_set_spec.cpp
View file

@ -1,68 +0,0 @@
#include "spec_helper.h"
#include <memory>
using namespace tree_sitter::lr;
using namespace tree_sitter::rules;
static item_set_ptr item_set(const std::initializer_list<Item> &items) {
return item_set_ptr(new ItemSet(items));
}
START_TEST
describe("item sets", []() {
Grammar grammar = test_grammars::arithmetic();
it("computes the closure of an item set under symbol expansion", [&]() {
Item item = Item::at_beginning_of_rule("expression", grammar);
ItemSet set = ItemSet(item, grammar);
AssertThat(
set,
EqualsContainer(ItemSet({
Item("expression", grammar.rule("expression"), 0),
Item("term", grammar.rule("term"), 0),
Item("factor", grammar.rule("factor"), 0),
Item("variable", grammar.rule("variable"), 0),
Item("number", grammar.rule("number"), 0),
Item("left_paren", grammar.rule("left_paren"), 0),
})));
});
it("computes transitions", [&]() {
Item item = Item::at_beginning_of_rule("factor", grammar);
ItemSet set = ItemSet(item, grammar);
AssertThat(
set.sym_transitions(grammar),
Equals(transition_map<rules::Symbol, ItemSet>({
{ sym("variable"), item_set({ Item("factor", blank(), 1) }) },
{ sym("number"), item_set({ Item("factor", blank(), 1) }) },
{ sym("left_paren"), std::make_shared<ItemSet>(Item("factor", seq({ sym("expression"), sym("right_paren") }), 1), grammar) },
})));
});
it("computes character transitions", [&]() {
Item item = Item::at_beginning_of_rule("factor", grammar);
ItemSet set = ItemSet(item, grammar);
AssertThat(
set.char_transitions(grammar),
Equals(transition_map<rules::Character, ItemSet>({
{ character(CharClassWord), item_set({ Item("variable", choice({ repeat(character(CharClassWord)), blank() }), 1) }) },
{ character(CharClassDigit), item_set({ Item("number", choice({ repeat(character(CharClassDigit)), blank() }), 1) }) },
{ character('('), item_set({ Item("left_paren", blank(), 1) }) }
})));
});
it("can be hashed", [&]() {
ItemSet set1 = ItemSet(Item::at_beginning_of_rule("factor", grammar), grammar);
ItemSet set2 = ItemSet(Item::at_beginning_of_rule("factor", grammar), grammar);
AssertThat(std::hash<const ItemSet>()(set1), Equals(std::hash<const ItemSet>()(set2)));
ItemSet set3 = ItemSet(Item::at_beginning_of_rule("term", grammar), grammar);
AssertThat(std::hash<const ItemSet>()(set1), !Equals(std::hash<const ItemSet>()(set3)));
});
});
END_TEST

61
spec/compiler/lr/table_builder_spec.cpp
View file

@ -2,15 +2,44 @@
#include <functional>
using namespace tree_sitter::lr;
using namespace tree_sitter::rules;
typedef std::unordered_set<ParseAction> parse_actions;
typedef std::unordered_set<LexAction> lex_actions;
START_TEST
describe("building parse and lex tables", []() {
Grammar grammar = test_grammars::arithmetic();
ParseTable table = build_tables(grammar).first;
LexTable lex_table = build_tables(grammar).second;
Grammar grammar({
{ "expression", choice({
seq({
sym("term"),
token("plus-token"),
sym("term") }),
sym("term") }) },
{ "term", choice({
sym("variable"),
sym("number"),
seq({
token("left-paren-token"),
sym("expression"),
token("right-paren-token")
}) }) },
{ "variable", token("variable-token") },
{ "number", token("number-token") }
});
Grammar lex_grammar({
{ "plus-token", character('+') },
{ "variable-token", pattern("\\w+") },
{ "number-token", pattern("\\d+") },
{ "left-paren-token", character('(') },
{ "right-paren-token", character(')') }
});
pair<ParseTable, LexTable> tables = build_tables(grammar, lex_grammar);
ParseTable table = tables.first;
LexTable lex_table = tables.second;
function<ParseState(size_t)> parse_state = [&](size_t index) {
return table.states[index];
@ -25,16 +54,18 @@ describe("building parse and lex tables", []() {
AssertThat(parse_state(0).actions, Equals(unordered_map<string, parse_actions>({
{ "expression", parse_actions({ ParseAction::Shift(1) }) },
{ "term", parse_actions({ ParseAction::Shift(2) }) },
{ "factor", parse_actions({ ParseAction::Shift(5) }) },
{ "variable", parse_actions({ ParseAction::Shift(8) }) },
{ "number", parse_actions({ ParseAction::Shift(8) }) },
{ "left_paren", parse_actions({ ParseAction::Shift(9) }) }
{ "number", parse_actions({ ParseAction::Shift(5) }) },
{ "variable", parse_actions({ ParseAction::Shift(5) }) },
{ "left-paren-token", parse_actions({ ParseAction::Shift(6) }) },
{ "variable-token", parse_actions({ ParseAction::Shift(9) }) },
{ "number-token", parse_actions({ ParseAction::Shift(10) }) },
})));
AssertThat(lex_state(0).actions, Equals(unordered_map<CharMatch, lex_actions>({
{ CharMatchClass(CharClassWord), lex_actions({ LexAction::Advance(1) }) },
{ CharMatchClass(CharClassDigit), lex_actions({ LexAction::Advance(4) }) },
{ CharMatchSpecific('('), lex_actions({ LexAction::Advance(11) }) }
{ CharMatchSpecific('('), lex_actions({ LexAction::Advance(1) }) },
{ CharMatchClass(CharClassWord), lex_actions({ LexAction::Advance(2) }) },
{ CharMatchClass(CharClassDigit), lex_actions({ LexAction::Advance(3) }) },
})));
});
@ -46,15 +77,7 @@ describe("building parse and lex tables", []() {
it("has the right next states", [&]() {
AssertThat(parse_state(2).actions, Equals(unordered_map<string, parse_actions>({
{ "plus", parse_actions({ ParseAction::Shift(3) }) },
})));
AssertThat(parse_state(3).actions, Equals(unordered_map<string, parse_actions>({
{ "variable", parse_actions({ ParseAction::Shift(8) }) },
{ "factor", parse_actions({ ParseAction::Shift(5) }) },
{ "left_paren", parse_actions({ ParseAction::Shift(9) }) },
{ "number", parse_actions({ ParseAction::Shift(8) }) },
{ "term", parse_actions({ ParseAction::Shift(4) }) },
{ "plus-token", parse_actions({ ParseAction::Shift(3) }) },
})));
});
});

1
spec/compiler/spec_helper.h
View file

@ -2,6 +2,7 @@
#define TreeSitter_SpecHelper_h
#include "bandit/bandit.h"
#include <iostream>
#include "transition_map.h"
#include "rules.h"
#include "item.h"

12
spec/fixtures/grammars/arithmetic.cpp vendored
View file

@ -10,28 +10,24 @@ namespace test_grammars {
{ "expression", choice({
seq({
sym("term"),
sym("plus"),
character('+'),
sym("term") }),
sym("term") }) },
{ "term", choice({
seq({
sym("factor"),
sym("times"),
character('*'),
sym("factor") }),
sym("factor") }) },
{ "factor", choice({
sym("variable"),
sym("number"),
seq({
sym("left_paren"),
character('('),
sym("expression"),
sym("right_paren") }) }) },
character(')') }) }) },
{ "number", pattern("\\d+") },
{ "variable", pattern("\\w+") },
{ "plus", str("+") },
{ "times", str("*") },
{ "left_paren", str("(") },
{ "right_paren", str(")") }
});
}
}

184
spec/fixtures/parsers/arithmetic.c vendored
View file

@ -4,13 +4,15 @@
typedef enum {
ts_symbol_expression,
ts_symbol_term,
ts_symbol_right_paren,
ts_symbol_number,
ts_symbol_factor,
ts_symbol_variable,
ts_symbol_plus,
ts_symbol_times,
ts_symbol_left_paren,
ts_symbol_6,
ts_symbol_5,
ts_symbol_4,
ts_symbol_3,
ts_symbol_2,
ts_symbol_1,
ts_symbol___END__
} ts_symbol;
@ -18,107 +20,43 @@ static void ts_lex(TSParser *parser) {
START_LEXER();
switch (LEX_STATE()) {
case 0:
if (LOOKAHEAD_CHAR() == '(')
ADVANCE(11);
if (isdigit(LOOKAHEAD_CHAR()))
ADVANCE(4);
if (isalnum(LOOKAHEAD_CHAR()))
ADVANCE(2);
if (isdigit(LOOKAHEAD_CHAR()))
ADVANCE(3);
if (LOOKAHEAD_CHAR() == '(')
ADVANCE(1);
LEX_ERROR();
case 1:
if (isalnum(LOOKAHEAD_CHAR()))
ADVANCE(2);
LEX_ERROR();
ACCEPT_TOKEN(ts_symbol_2);
case 2:
if (isalnum(LOOKAHEAD_CHAR()))
ADVANCE(3);
LEX_ERROR();
ADVANCE(2);
ACCEPT_TOKEN(ts_symbol_1);
case 3:
if (isalnum(LOOKAHEAD_CHAR()))
ADVANCE(1);
LEX_ERROR();
case 4:
if (isdigit(LOOKAHEAD_CHAR()))
ADVANCE(5);
ADVANCE(3);
ACCEPT_TOKEN(ts_symbol_4);
case 4:
LEX_ERROR();
case 5:
if (isdigit(LOOKAHEAD_CHAR()))
if (LOOKAHEAD_CHAR() == '+')
ADVANCE(6);
LEX_ERROR();
case 6:
if (isdigit(LOOKAHEAD_CHAR()))
ADVANCE(7);
LEX_ERROR();
ACCEPT_TOKEN(ts_symbol_6);
case 7:
if (isdigit(LOOKAHEAD_CHAR()))
if (LOOKAHEAD_CHAR() == '*')
ADVANCE(8);
LEX_ERROR();
case 8:
if (isdigit(LOOKAHEAD_CHAR()))
ADVANCE(9);
LEX_ERROR();
ACCEPT_TOKEN(ts_symbol_5);
case 9:
if (isdigit(LOOKAHEAD_CHAR()))
if (LOOKAHEAD_CHAR() == ')')
ADVANCE(10);
LEX_ERROR();
case 10:
if (isdigit(LOOKAHEAD_CHAR()))
ADVANCE(8);
LEX_ERROR();
case 11:
ACCEPT_TOKEN(ts_symbol_left_paren);
case 12:
ACCEPT_TOKEN(ts_symbol___START__);
case 13:
if (LOOKAHEAD_CHAR() == '+')
ADVANCE(14);
LEX_ERROR();
case 14:
ACCEPT_TOKEN(ts_symbol_plus);
case 15:
if (LOOKAHEAD_CHAR() == '(')
ADVANCE(11);
if (isdigit(LOOKAHEAD_CHAR()))
ADVANCE(4);
if (isalnum(LOOKAHEAD_CHAR()))
ADVANCE(1);
LEX_ERROR();
case 16:
ACCEPT_TOKEN(ts_symbol_expression);
case 17:
if (LOOKAHEAD_CHAR() == '*')
ADVANCE(18);
LEX_ERROR();
case 18:
ACCEPT_TOKEN(ts_symbol_times);
case 19:
if (LOOKAHEAD_CHAR() == '(')
ADVANCE(11);
if (isdigit(LOOKAHEAD_CHAR()))
ADVANCE(4);
if (isalnum(LOOKAHEAD_CHAR()))
ADVANCE(1);
LEX_ERROR();
case 20:
ACCEPT_TOKEN(ts_symbol_term);
case 21:
ACCEPT_TOKEN(ts_symbol_factor);
case 22:
if (LOOKAHEAD_CHAR() == '(')
ADVANCE(11);
if (isdigit(LOOKAHEAD_CHAR()))
ADVANCE(4);
if (isalnum(LOOKAHEAD_CHAR()))
ADVANCE(1);
LEX_ERROR();
case 23:
if (LOOKAHEAD_CHAR() == ')')
ADVANCE(24);
LEX_ERROR();
case 24:
ACCEPT_TOKEN(ts_symbol_right_paren);
case 25:
ACCEPT_TOKEN(ts_symbol_factor);
ACCEPT_TOKEN(ts_symbol_3);
default:
LEX_ERROR();
}
@ -131,14 +69,18 @@ TSTree ts_parse_arithmetic(const char *input) {
case 0:
SET_LEX_STATE(0);
switch (LOOKAHEAD_SYM()) {
case ts_symbol_left_paren:
SHIFT(9);
case ts_symbol_4:
SHIFT(13);
case ts_symbol_variable:
SHIFT(8);
case ts_symbol_factor:
SHIFT(5);
case ts_symbol_number:
SHIFT(8);
case ts_symbol_1:
SHIFT(12);
case ts_symbol_2:
SHIFT(9);
case ts_symbol_term:
SHIFT(2);
case ts_symbol_expression:
@ -147,7 +89,7 @@ TSTree ts_parse_arithmetic(const char *input) {
PARSE_ERROR();
}
case 1:
SET_LEX_STATE(12);
SET_LEX_STATE(4);
switch (LOOKAHEAD_SYM()) {
case ts_symbol___END__:
ACCEPT_INPUT();
@ -155,48 +97,56 @@ TSTree ts_parse_arithmetic(const char *input) {
PARSE_ERROR();
}
case 2:
SET_LEX_STATE(13);
SET_LEX_STATE(5);
switch (LOOKAHEAD_SYM()) {
case ts_symbol_plus:
case ts_symbol_6:
SHIFT(3);
default:
PARSE_ERROR();
REDUCE(ts_symbol_expression, 1);
}
case 3:
SET_LEX_STATE(15);
SET_LEX_STATE(0);
switch (LOOKAHEAD_SYM()) {
case ts_symbol_4:
SHIFT(13);
case ts_symbol_1:
SHIFT(12);
case ts_symbol_term:
SHIFT(4);
case ts_symbol_2:
SHIFT(9);
case ts_symbol_variable:
SHIFT(8);
case ts_symbol_factor:
SHIFT(5);
case ts_symbol_left_paren:
SHIFT(9);
case ts_symbol_number:
SHIFT(8);
case ts_symbol_term:
SHIFT(4);
default:
PARSE_ERROR();
}
case 4:
SET_LEX_STATE(16);
SET_LEX_STATE(4);
switch (LOOKAHEAD_SYM()) {
default:
REDUCE(ts_symbol_expression, 3);
}
case 5:
SET_LEX_STATE(17);
SET_LEX_STATE(7);
switch (LOOKAHEAD_SYM()) {
case ts_symbol_times:
case ts_symbol_5:
SHIFT(6);
default:
PARSE_ERROR();
REDUCE(ts_symbol_term, 1);
}
case 6:
SET_LEX_STATE(19);
SET_LEX_STATE(0);
switch (LOOKAHEAD_SYM()) {
case ts_symbol_left_paren:
case ts_symbol_4:
SHIFT(13);
case ts_symbol_2:
SHIFT(9);
case ts_symbol_1:
SHIFT(12);
case ts_symbol_number:
SHIFT(8);
case ts_symbol_variable:
@ -207,28 +157,32 @@ TSTree ts_parse_arithmetic(const char *input) {
PARSE_ERROR();
}
case 7:
SET_LEX_STATE(20);
SET_LEX_STATE(4);
switch (LOOKAHEAD_SYM()) {
default:
REDUCE(ts_symbol_term, 3);
}
case 8:
SET_LEX_STATE(21);
SET_LEX_STATE(4);
switch (LOOKAHEAD_SYM()) {
default:
REDUCE(ts_symbol_factor, 1);
}
case 9:
SET_LEX_STATE(22);
SET_LEX_STATE(0);
switch (LOOKAHEAD_SYM()) {
case ts_symbol_left_paren:
SHIFT(9);
case ts_symbol_4:
SHIFT(13);
case ts_symbol_variable:
SHIFT(8);
case ts_symbol_factor:
SHIFT(5);
case ts_symbol_number:
SHIFT(8);
case ts_symbol_1:
SHIFT(12);
case ts_symbol_2:
SHIFT(9);
case ts_symbol_term:
SHIFT(2);
case ts_symbol_expression:
@ -237,19 +191,31 @@ TSTree ts_parse_arithmetic(const char *input) {
PARSE_ERROR();
}
case 10:
SET_LEX_STATE(23);
SET_LEX_STATE(9);
switch (LOOKAHEAD_SYM()) {
case ts_symbol_right_paren:
case ts_symbol_3:
SHIFT(11);
default:
PARSE_ERROR();
}
case 11:
SET_LEX_STATE(25);
SET_LEX_STATE(4);
switch (LOOKAHEAD_SYM()) {
default:
REDUCE(ts_symbol_factor, 3);
}
case 12:
SET_LEX_STATE(4);
switch (LOOKAHEAD_SYM()) {
default:
REDUCE(ts_symbol_variable, 1);
}
case 13:
SET_LEX_STATE(4);
switch (LOOKAHEAD_SYM()) {
default:
REDUCE(ts_symbol_number, 1);
}
default:
PARSE_ERROR();
}

9
spec/main.cpp
View file

@ -2,6 +2,11 @@
int main(int argc, char *argv[])
{
char *args[] = {nullptr, (char *)"--no-color"};
return bandit::run(2, args);
const char *args[] = {
"",
"--no-color",
"--only="
"",
};
return bandit::run(4, const_cast<char **>(args));
}

12
src/compiler/code_gen/c_code.cpp
View file

@ -51,12 +51,12 @@ namespace tree_sitter {
}
class CCodeGenerator {
const Grammar grammar;
const vector<string> rule_names;
const ParseTable parse_table;
const LexTable lex_table;
public:
CCodeGenerator(const Grammar &grammar, const ParseTable &parse_table, const LexTable &lex_table) :
grammar(grammar),
CCodeGenerator(vector<string> rule_names, const ParseTable &parse_table, const LexTable &lex_table) :
rule_names(rule_names),
parse_table(parse_table),
lex_table(lex_table)
{}
@ -152,7 +152,7 @@ namespace tree_sitter {
string symbol_enum() {
string result = "typedef enum {\n";
for (string rule_name : grammar.rule_names())
for (string rule_name : rule_names)
result += indent(symbol_id(rule_name)) + ",\n";
result += indent(symbol_id(ParseTable::END_OF_INPUT));
return result + "\n"
@ -192,8 +192,8 @@ namespace tree_sitter {
}
};
string c_code(const Grammar &grammar, const ParseTable &parse_table, const LexTable &lex_table) {
return CCodeGenerator(grammar, parse_table, lex_table).code();
string c_code(const vector<string> rule_names, const ParseTable &parse_table, const LexTable &lex_table) {
return CCodeGenerator(rule_names, parse_table, lex_table).code();
}
}
}

2
src/compiler/code_gen/c_code.h
View file

@ -7,7 +7,7 @@
namespace tree_sitter {
namespace code_gen {
std::string c_code(const Grammar &grammar, const lr::ParseTable &parse_table, const lr::LexTable &lex_table);
std::string c_code(std::vector<std::string> rule_names, const lr::ParseTable &parse_table, const lr::LexTable &lex_table);
}
}

20
src/compiler/lr/item.cpp
View file

@ -16,17 +16,22 @@ namespace tree_sitter {
Item Item::at_beginning_of_rule(const std::string &rule_name, const Grammar &grammar) {
return Item(rule_name, grammar.rule(rule_name), 0);
}
Item Item::at_beginning_of_token(const std::string &rule_name, const Grammar &grammar) {
return Item(rule_name, grammar.rule(rule_name), -1);
}
transition_map<rules::Rule, Item> Item::transitions() const {
return lr::transitions(rule).map<Item>([&](rules::rule_ptr to_rule) -> item_ptr {
return std::make_shared<Item>(rule_name, to_rule, consumed_sym_count + 1);
int next_sym_count = (consumed_sym_count == -1) ? -1 : (consumed_sym_count + 1);
return std::make_shared<Item>(rule_name, to_rule, next_sym_count);
});
};
vector<rules::Symbol> Item::next_symbols() const {
vector<rules::Symbol> result;
vector<rules::NonTerminal> Item::next_symbols() const {
vector<rules::NonTerminal> result;
for (auto pair : lr::transitions(rule)) {
shared_ptr<const rules::Symbol> sym = dynamic_pointer_cast<const rules::Symbol>(pair.first);
auto sym = dynamic_pointer_cast<const rules::NonTerminal>(pair.first);
if (sym) result.push_back(*sym);
}
return result;
@ -39,7 +44,10 @@ namespace tree_sitter {
}
bool Item::is_done() const {
return *rule == rules::Blank();
for (auto pair : transitions()) {
if (*pair.first == rules::Blank()) return true;
}
return false;
}
std::ostream& operator<<(ostream &stream, const Item &item) {

5
src/compiler/lr/item.h
View file

@ -3,7 +3,7 @@
#include <string>
#include "rule.h"
#include "symbol.h"
#include "non_terminal.h"
#include "transition_map.h"
namespace tree_sitter {
@ -17,9 +17,10 @@ namespace tree_sitter {
public:
Item(const std::string &rule_name, const rules::rule_ptr rule, int consumed_sym_count);
static Item at_beginning_of_rule(const std::string &rule_name, const Grammar &grammar);
static Item at_beginning_of_token(const std::string &rule_name, const Grammar &grammar);
transition_map<rules::Rule, Item> transitions() const;
std::vector<rules::Symbol> next_symbols() const;
std::vector<rules::NonTerminal> next_symbols() const;
bool operator==(const Item &other) const;
bool is_done() const;

22
src/compiler/lr/item_set.cpp
View file

@ -18,7 +18,7 @@ namespace tree_sitter {
static void add_item(vector<Item> &vector, const Item &item, const Grammar &grammar) {
if (!vector_contains(vector, item)) {
vector.push_back(item);
for (rules::Symbol rule : item.next_symbols()) {
for (rules::NonTerminal rule : item.next_symbols()) {
Item next_item = Item::at_beginning_of_rule(rule.name, grammar);
add_item(vector, next_item, grammar);
}
@ -33,29 +33,17 @@ namespace tree_sitter {
ItemSet::ItemSet(const Item &item, const Grammar &grammar) : contents(closure_in_grammar(item, grammar)) {}
template<typename RuleClass>
static transition_map<RuleClass, ItemSet> transitions(const ItemSet &item_set, const Grammar &grammar) {
transition_map<RuleClass, ItemSet> result;
for (auto item : item_set) {
transition_map<rules::Rule, ItemSet> ItemSet::all_transitions(const Grammar &grammar) const {
transition_map<rules::Rule, ItemSet> result;
for (auto item : *this) {
auto item_transitions = item.transitions();
for (auto pair : item_transitions) {
std::shared_ptr<const RuleClass> rule = dynamic_pointer_cast<const RuleClass>(pair.first);
Item item = *pair.second;
if (rule.get() != nullptr)
result.add(rule, std::make_shared<ItemSet>(item, grammar));
result.add(pair.first, std::make_shared<ItemSet>(*pair.second, grammar));
}
}
return result;
}
transition_map<rules::Character, ItemSet> ItemSet::char_transitions(const Grammar &grammar) const {
return transitions<rules::Character>(*this, grammar);
}
transition_map<rules::Symbol, ItemSet> ItemSet::sym_transitions(const Grammar &grammar) const {
return transitions<rules::Symbol>(*this, grammar);
}
bool ItemSet::operator==(const tree_sitter::lr::ItemSet &other) const {
return contents == other.contents;
}

21
src/compiler/lr/item_set.h
View file

@ -21,9 +21,26 @@ namespace tree_sitter {
const_iterator begin() const;
const_iterator end() const;
size_t size() const;
transition_map<rules::Rule, ItemSet> all_transitions(const Grammar &grammar) const;
transition_map<rules::Character, ItemSet> char_transitions(const Grammar &grammar) const;
transition_map<rules::Symbol, ItemSet> sym_transitions(const Grammar &grammar) const;
template<typename RuleClass>
transition_map<RuleClass, ItemSet> transitions(const Grammar &grammar) const {
transition_map<RuleClass, ItemSet> result;
for (auto transition : all_transitions(grammar)) {
auto rule = std::dynamic_pointer_cast<const RuleClass>(transition.first);
if (rule.get()) result.add(rule, transition.second);
}
return result;
}
template<typename RuleClass>
std::vector<RuleClass> next_inputs(const Grammar &grammar) const {
std::vector<RuleClass> result;
for (auto pair : transitions<RuleClass>(grammar))
result.push_back(*pair.first);
return result;
}
bool operator==(const ItemSet &other) const;
};

2
src/compiler/lr/lex_table.cpp
View file

@ -49,8 +49,6 @@ namespace tree_sitter {
LexState::LexState() : actions(unordered_map<CharMatch, unordered_set<LexAction>>()) {}
// Table
LexTable::LexTable(vector<string> rule_names) : symbol_names(rule_names) {}
size_t LexTable::add_state() {
states.push_back(LexState());
return states.size() - 1;

3
src/compiler/lr/lex_table.h
View file

@ -55,8 +55,6 @@ namespace tree_sitter {
class LexTable {
public:
LexTable(std::vector<std::string> rule_names);
size_t add_state();
void add_action(size_t state_index, CharMatch match, LexAction action);
void add_default_action(size_t state_index, LexAction action);
@ -64,7 +62,6 @@ namespace tree_sitter {
static const std::string START;
static const std::string END_OF_INPUT;
std::vector<LexState> states;
const std::vector<std::string> symbol_names;
};
}
}

4
src/compiler/lr/parse_table.cpp
View file

@ -55,10 +55,6 @@ namespace tree_sitter {
{}
// Table
ParseTable::ParseTable(vector<string> symbol_names) :
symbol_names(symbol_names),
states(vector<ParseState>()) {};
size_t ParseTable::add_state() {
states.push_back(ParseState());
return states.size() - 1;

3
src/compiler/lr/parse_table.h
View file

@ -59,8 +59,6 @@ namespace tree_sitter {
class ParseTable {
public:
ParseTable(std::vector<std::string> rule_names);
size_t add_state();
void add_action(size_t state_index, std::string symbol_name, ParseAction action);
void add_default_action(size_t state_index, ParseAction action);
@ -68,7 +66,6 @@ namespace tree_sitter {
static const std::string START;
static const std::string END_OF_INPUT;
std::vector<ParseState> states;
const std::vector<std::string> symbol_names;
};
}
}

29
src/compiler/lr/table_builder.cpp
View file

@ -5,6 +5,8 @@
#include "item_set.h"
#include "grammar.h"
#include <iostream>
using namespace std;
namespace tree_sitter {
@ -13,6 +15,7 @@ namespace tree_sitter {
class TableBuilder {
const Grammar grammar;
const Grammar lex_grammar;
std::unordered_map<const ItemSet, size_t> parse_state_indices;
std::unordered_map<const ItemSet, size_t> lex_state_indices;
ParseTable parse_table;
@ -29,7 +32,7 @@ namespace tree_sitter {
}
void add_shift_actions(const ItemSet &item_set, size_t state_index) {
for (auto transition : item_set.sym_transitions(grammar)) {
for (auto transition : item_set.transitions<rules::Symbol>(grammar)) {
rules::Symbol symbol = *transition.first;
ItemSet item_set = *transition.second;
size_t new_state_index = add_parse_state(item_set);
@ -38,7 +41,7 @@ namespace tree_sitter {
}
void add_advance_actions(const ItemSet &item_set, size_t state_index) {
for (auto transition : item_set.char_transitions(grammar)) {
for (auto transition : item_set.transitions<rules::Character>(grammar)) {
rules::Character rule = *transition.first;
ItemSet item_set = *transition.second;
size_t new_state_index = add_lex_state(item_set);
@ -77,13 +80,21 @@ namespace tree_sitter {
return state_index;
}
ItemSet lex_item_set_for_parse_item_set(const ItemSet &parse_item_set) {
vector<Item> items;
for (rules::Token token : parse_item_set.next_inputs<rules::Token>(grammar))
items.push_back(Item::at_beginning_of_token(token.name, lex_grammar));
return ItemSet(items);
}
size_t add_parse_state(const ItemSet &item_set) {
auto state_index = parse_state_index_for_item_set(item_set);
if (state_index == NOT_FOUND) {
state_index = parse_table.add_state();
parse_state_indices[item_set] = state_index;
parse_table.states[state_index].lex_state_index = add_lex_state(item_set);
ItemSet lex_item_set = lex_item_set_for_parse_item_set(item_set);
parse_table.states[state_index].lex_state_index = add_lex_state(lex_item_set);
add_shift_actions(item_set, state_index);
add_reduce_actions(item_set, state_index);
}
@ -92,13 +103,9 @@ namespace tree_sitter {
public:
TableBuilder(const Grammar &grammar) :
TableBuilder(const Grammar &grammar, const Grammar &lex_grammar) :
grammar(grammar),
parse_table(ParseTable(grammar.rule_names())),
lex_table(LexTable(grammar.rule_names())),
parse_state_indices(unordered_map<const ItemSet, size_t>()),
lex_state_indices(unordered_map<const ItemSet, size_t>())
{};
lex_grammar(lex_grammar) {};
std::pair<ParseTable, LexTable> build() {
auto item = Item(ParseTable::START, rules::sym(grammar.start_rule_name), 0);
@ -108,8 +115,8 @@ namespace tree_sitter {
}
};
std::pair<ParseTable, LexTable> build_tables(const tree_sitter::Grammar &grammar) {
return TableBuilder(grammar).build();
std::pair<ParseTable, LexTable> build_tables(const Grammar &grammar, const Grammar &lex_grammar) {
return TableBuilder(grammar, lex_grammar).build();
}
}
}

2
src/compiler/lr/table_builder.h
View file

@ -8,7 +8,7 @@ namespace tree_sitter {
class Grammar;
namespace lr {
std::pair<ParseTable, LexTable> build_tables(const Grammar &grammar);
std::pair<ParseTable, LexTable> build_tables(const Grammar &grammar, const Grammar &lex_grammar);
}
}

10
src/compiler/lr/transitions.cpp
View file

@ -10,7 +10,7 @@ namespace tree_sitter {
transition_map<Rule, Rule> value;
void visit(const Blank *rule) {
value = transition_map<Rule, Rule>();
value = transition_map<Rule, Rule>({{ blank(), blank() }});
}
void visit(const Character *rule) {
@ -20,7 +20,11 @@ namespace tree_sitter {
void visit(const Symbol *rule) {
value = transition_map<Rule, Rule>({{ rule->copy(), blank() }});
}
void visit(const Token *rule) {
value = transition_map<Rule, Rule>({{ rule->copy(), blank() }});
}
void visit(const Choice *rule) {
value = transitions(rule->left);
value.merge(transitions(rule->right), [&](rule_ptr left, rule_ptr right) -> rule_ptr {
@ -39,7 +43,7 @@ namespace tree_sitter {
void visit(const Repeat *rule) {
value = transitions(rule->content).map<Rule>([&](const rule_ptr &value) -> rule_ptr {
return seq({ value, choice({ repeat(rule->content), blank() }) });
return seq({ value, choice({ rule->copy(), blank() }) });
});
}

28
src/compiler/rules/non_terminal.cpp Normal file
View file

@ -0,0 +1,28 @@
#include "rules.h"
#include "transition_map.h"
using std::string;
using std::hash;
namespace tree_sitter {
namespace rules {
NonTerminal::NonTerminal(const std::string &name) : Symbol(name) {};
bool NonTerminal::operator==(const Rule &rule) const {
const NonTerminal *other = dynamic_cast<const NonTerminal *>(&rule);
return other && (other->name == name);
}
rule_ptr NonTerminal::copy() const {
return std::make_shared<NonTerminal>(*this);
}
string NonTerminal::to_string() const {
return string("#<non-terminal '") + name + "'>";
}
void NonTerminal::accept(Visitor &visitor) const {
visitor.visit(this);
}
}
}

20
src/compiler/rules/non_terminal.h Normal file
View file

@ -0,0 +1,20 @@
#ifndef __tree_sitter__non_terminal__
#define __tree_sitter__non_terminal__
#include "symbol.h"
namespace tree_sitter {
namespace rules {
class NonTerminal : public Symbol {
public:
NonTerminal(const std::string &name);
bool operator==(const Rule& other) const;
rule_ptr copy() const;
std::string to_string() const;
void accept(Visitor &visitor) const;
};
}
}
#endif

2
src/compiler/rules/rules.cpp
View file

@ -47,7 +47,7 @@ namespace tree_sitter {
}
sym_ptr sym(const string &name) {
return make_shared<Symbol>(name);
return make_shared<NonTerminal>(name);
}
rule_ptr token(const std::string &name) {

1
src/compiler/rules/rules.h
View file

@ -11,6 +11,7 @@
#include "pattern.h"
#include "character.h"
#include "repeat.h"
#include "non_terminal.h"
#include "visitor.h"
namespace tree_sitter {

6
src/compiler/rules/token.cpp
View file

@ -6,17 +6,13 @@ using std::hash;
namespace tree_sitter {
namespace rules {
Token::Token(const std::string &name) : name(name) {};
Token::Token(const std::string &name) : Symbol(name) {};
bool Token::operator==(const Rule &rule) const {
const Token *other = dynamic_cast<const Token *>(&rule);
return other && (other->name == name);
}
size_t Token::hash_code() const {
return typeid(this).hash_code() ^ hash<string>()(name);
}
rule_ptr Token::copy() const {
return std::make_shared<Token>(*this);
}

7
src/compiler/rules/token.h
View file

@ -1,21 +1,18 @@
#ifndef __tree_sitter__token__
#define __tree_sitter__token__
#include "rule.h"
#include "symbol.h"
namespace tree_sitter {
namespace rules {
class Token : public Rule {
class Token : public Symbol {
public:
Token(const std::string &name);
bool operator==(const Rule& other) const;
size_t hash_code() const;
rule_ptr copy() const;
std::string to_string() const;
void accept(Visitor &visitor) const;
const std::string name;
};
}
}

12
tree_sitter.xcodeproj/project.pbxproj
View file

@ -16,6 +16,8 @@
1213061B182C84DF00FCF928 /* item.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 12130619182C84DF00FCF928 /* item.cpp */; };
12130622182C85D300FCF928 /* item_set.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 12130620182C85D300FCF928 /* item_set.cpp */; };
1214930E181E200B008E9BDA /* main.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 121492E9181E200B008E9BDA /* main.cpp */; };
121D8B2E187763F3003CF44B /* non_terminal.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 121D8B2C187763F3003CF44B /* non_terminal.cpp */; };
121D8B2F1877AD1C003CF44B /* non_terminal.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 121D8B2C187763F3003CF44B /* non_terminal.cpp */; };
1225CC6418765693000D4723 /* prepare_grammar_spec.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1225CC6318765693000D4723 /* prepare_grammar_spec.cpp */; };
1225CC6718765737000D4723 /* prepare_grammar.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1225CC6518765737000D4723 /* prepare_grammar.cpp */; };
1225CC6A187661C7000D4723 /* extract_tokens.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1225CC68187661C7000D4723 /* extract_tokens.cpp */; };
@ -50,7 +52,6 @@
12FD40CA185EEB5E0041A84E /* rule.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1251209A1830145300C9B56A /* rule.cpp */; };
12FD40CB185EEB5E0041A84E /* pattern.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27A340F3EEB184C040521323 /* pattern.cpp */; };
12FD40D2185EEB970041A84E /* arithmetic.c in Sources */ = {isa = PBXBuildFile; fileRef = 12FD4065185E7C2F0041A84E /* arithmetic.c */; };
12FD40D5185FEEDB0041A84E /* item_set_spec.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1213061D182C857100FCF928 /* item_set_spec.cpp */; };
12FD40D6185FEEDB0041A84E /* table_builder_spec.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 12512092182F307C00C9B56A /* table_builder_spec.cpp */; };
12FD40D7185FEEDB0041A84E /* item_spec.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 12D1369C18328C5A005F3369 /* item_spec.cpp */; };
12FD40D8185FEEDF0041A84E /* rules_spec.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 121492EA181E200B008E9BDA /* rules_spec.cpp */; };
@ -108,11 +109,12 @@
12130616182C3D2900FCF928 /* string.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = string.h; sourceTree = "<group>"; };
12130619182C84DF00FCF928 /* item.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = item.cpp; sourceTree = "<group>"; };
1213061A182C84DF00FCF928 /* item.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = item.h; sourceTree = "<group>"; };
1213061D182C857100FCF928 /* item_set_spec.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = item_set_spec.cpp; path = spec/compiler/lr/item_set_spec.cpp; sourceTree = SOURCE_ROOT; };
12130620182C85D300FCF928 /* item_set.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = item_set.cpp; sourceTree = "<group>"; };
12130621182C85D300FCF928 /* item_set.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = item_set.h; sourceTree = "<group>"; };
121492E9181E200B008E9BDA /* main.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = main.cpp; path = spec/main.cpp; sourceTree = SOURCE_ROOT; };
121492EA181E200B008E9BDA /* rules_spec.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = rules_spec.cpp; path = spec/compiler/rules/rules_spec.cpp; sourceTree = SOURCE_ROOT; };
121D8B2C187763F3003CF44B /* non_terminal.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = non_terminal.cpp; sourceTree = "<group>"; };
121D8B2D187763F3003CF44B /* non_terminal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = non_terminal.h; sourceTree = "<group>"; };
1225CC6318765693000D4723 /* prepare_grammar_spec.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = prepare_grammar_spec.cpp; sourceTree = "<group>"; };
1225CC6518765737000D4723 /* prepare_grammar.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = prepare_grammar.cpp; sourceTree = "<group>"; };
1225CC6618765737000D4723 /* prepare_grammar.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = prepare_grammar.h; sourceTree = "<group>"; };
@ -210,6 +212,8 @@
1225CC6F1876AFFF000D4723 /* token.h */,
12FD40E618639B910041A84E /* visitor.cpp */,
12FD40E41862B3530041A84E /* visitor.h */,
121D8B2C187763F3003CF44B /* non_terminal.cpp */,
121D8B2D187763F3003CF44B /* non_terminal.h */,
);
path = rules;
sourceTree = "<group>";
@ -237,7 +241,6 @@
isa = PBXGroup;
children = (
12ED72A6186FC8220089229B /* transitions_spec.cpp */,
1213061D182C857100FCF928 /* item_set_spec.cpp */,
12512092182F307C00C9B56A /* table_builder_spec.cpp */,
12D1369C18328C5A005F3369 /* item_spec.cpp */,
);
@ -483,8 +486,8 @@
buildActionMask = 2147483647;
files = (
12FD40D7185FEEDB0041A84E /* item_spec.cpp in Sources */,
12FD40D5185FEEDB0041A84E /* item_set_spec.cpp in Sources */,
12130614182C3A1700FCF928 /* seq.cpp in Sources */,
121D8B2E187763F3003CF44B /* non_terminal.cpp in Sources */,
1225CC6A187661C7000D4723 /* extract_tokens.cpp in Sources */,
129D242C183EB1EB00FE9F71 /* table_builder.cpp in Sources */,
125120A4183083BD00C9B56A /* arithmetic.cpp in Sources */,
@ -527,6 +530,7 @@
12FD40B3185EEB5E0041A84E /* seq.cpp in Sources */,
12FD40B4185EEB5E0041A84E /* table_builder.cpp in Sources */,
12FD40B6185EEB5E0041A84E /* arithmetic.cpp in Sources */,
121D8B2F1877AD1C003CF44B /* non_terminal.cpp in Sources */,
12FD40DD185FF12C0041A84E /* parser.c in Sources */,
12FD40B8185EEB5E0041A84E /* item.cpp in Sources */,
12FD40B9185EEB5E0041A84E /* string.cpp in Sources */,