traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add rule precedence construct

Browse source 
Still need to add some way of expressing left and right
associativity
This commit is contained in:
Max Brunsfeld 2014-04-14 23:11:10 -07:00
parent e23604ac52
commit a437d39773
23 changed files with 750 additions and 1504 deletions
Download patch file Download diff file Expand all files Collapse all files

39
examples/grammars/arithmetic.cc
View file

@ -5,39 +5,34 @@ namespace tree_sitter_examples {
using namespace tree_sitter::rules;
using std::string;
static rule_ptr infix_op(string op, string rule_name) {
return choice({
seq({
sym(rule_name),
str(op),
sym(rule_name) }),
sym(rule_name) });
static rule_ptr infix_op(string op, int precedence) {
return prec(precedence, seq({
sym("expression"),
str(op),
sym("expression") }));
}
extern const Grammar arithmetic({
{ "expression", choice({
sym("sum"),
sym("difference") }) },
{ "_operand1", choice({
sym("difference"),
sym("product"),
sym("quotient") }) },
{ "_operand2", choice({
sym("exponent") }) },
{ "_operand3", choice({
sym("quotient"),
sym("exponent"),
sym("group"),
sym("number"),
sym("variable"),
sym("grouping") }) },
{ "grouping", seq({
sym("variable") }) },
{ "sum", infix_op("+", 1) },
{ "difference", infix_op("-", 1) },
{ "product", infix_op("*", 2) },
{ "quotient", infix_op("/", 2) },
{ "exponent", infix_op("^", 3) },
{ "group", seq({
str("("),
err(sym("expression")),
str(")") }) },
{ "sum", infix_op("+", "_operand1") },
{ "difference", infix_op("-", "_operand1") },
{ "product", infix_op("*", "_operand2") },
{ "quotient", infix_op("/", "_operand2") },
{ "exponent", infix_op("^", "_operand3") },
{ "number", pattern("\\d+") },
{ "variable", pattern("\\a[\\w_]*") },
});

1741
examples/parsers/arithmetic.c
View file

File diff suppressed because it is too large Load diff

1
include/tree_sitter/compiler.h
View file

@ -20,6 +20,7 @@ namespace tree_sitter {
rule_ptr pattern(const std::string &value);
rule_ptr str(const std::string &value);
rule_ptr err(const rule_ptr &rule);
rule_ptr prec(int precedence, rule_ptr rule);
}
class Grammar {

2
include/tree_sitter/parser.h
View file

@ -55,7 +55,7 @@ ts_lexer_start_token(lexer);
{ return ts_lexer_build_node(lexer, ts_builtin_sym_error); }
#define LEX_PANIC() \
{ DEBUG_LEX("\nLex error: unexpected state %d", lex_state); return NULL; }
{ DEBUG_LEX("LEX ERROR: unexpected state %d", lex_state); return NULL; }
#define PARSE_TABLE \
static const ts_parse_action ts_parse_actions[STATE_COUNT][SYMBOL_COUNT]

126
spec/compiler/build_tables/conflict_manager_spec.cc
View file

@ -59,49 +59,117 @@ describe("resolving parse conflicts", []() {
Symbol sym2("rule2");
it("favors non-errors over parse errors", [&]() {
should_update = manager->resolve_parse_action(sym1, ParseAction::Error(), ParseAction::Shift(2));
should_update = manager->resolve_parse_action(sym1, ParseAction::Error(), ParseAction::Shift(2, { 0 }));
AssertThat(should_update, IsTrue());
should_update = manager->resolve_parse_action(sym1, ParseAction::Shift(2), ParseAction::Error());
should_update = manager->resolve_parse_action(sym1, ParseAction::Shift(2, { 0 }), ParseAction::Error());
AssertThat(should_update, IsFalse());
});
describe("shift/reduce conflicts", [&]() {
it("records a conflict", [&]() {
manager->resolve_parse_action(sym1, ParseAction::Reduce(sym2, 1), ParseAction::Shift(2));
manager->resolve_parse_action(sym1, ParseAction::Shift(2), ParseAction::Reduce(sym2, 1));
AssertThat(manager->conflicts(), Equals(vector<Conflict>({
Conflict("rule1: shift / reduce rule2")
})));
describe("when the shift has higher precedence", [&]() {
ParseAction shift = ParseAction::Shift(2, { 3 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 1);
it("does not record a conflict", [&]() {
manager->resolve_parse_action(sym1, shift, reduce);
manager->resolve_parse_action(sym1, reduce, shift);
AssertThat(manager->conflicts(), IsEmpty());
});
it("favors the shift", [&]() {
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsTrue());
});
});
describe("when the reduce has higher precedence", [&]() {
ParseAction shift = ParseAction::Shift(2, { 1 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 3);
it("does not record a conflict", [&]() {
manager->resolve_parse_action(sym1, reduce, shift);
manager->resolve_parse_action(sym1, shift, reduce);
AssertThat(manager->conflicts(), IsEmpty());
});
it("favors the reduce", [&]() {
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsTrue());
});
});
describe("when the precedences are equal", [&]() {
ParseAction shift = ParseAction::Shift(2, { 0 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 0);
it("favors the shift", [&]() {
should_update = manager->resolve_parse_action(sym1, ParseAction::Reduce(sym2, 1), ParseAction::Shift(2));
AssertThat(should_update, IsTrue());
should_update = manager->resolve_parse_action(sym1, ParseAction::Shift(2), ParseAction::Reduce(sym2, 1));
AssertThat(should_update, IsFalse());
it("records a conflict", [&]() {
manager->resolve_parse_action(sym1, reduce, shift);
manager->resolve_parse_action(sym1, shift, reduce);
AssertThat(manager->conflicts(), Equals(vector<Conflict>({
Conflict("rule1: shift (precedence 0) / reduce rule2 (precedence 0)")
})));
});
it("favors the shift", [&]() {
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsTrue());
});
});
describe("when the shift has conflicting precedences compared to the reduce", [&]() {
ParseAction shift = ParseAction::Shift(2, { 0, 1, 3 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 2);
it("records a conflict", [&]() {
manager->resolve_parse_action(sym1, reduce, shift);
manager->resolve_parse_action(sym1, shift, reduce);
AssertThat(manager->conflicts(), Equals(vector<Conflict>({
Conflict("rule1: shift (precedence 0, 1, 3) / reduce rule2 (precedence 2)")
})));
});
it("favors the shift", [&]() {
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsTrue());
});
});
});
describe("reduce/reduce conflicts", [&]() {
it("records a conflict", [&]() {
manager->resolve_parse_action(sym1, ParseAction::Reduce(sym2, 1), ParseAction::Reduce(sym1, 1));
manager->resolve_parse_action(sym1, ParseAction::Reduce(sym1, 1), ParseAction::Reduce(sym2, 1));
AssertThat(manager->conflicts(), Equals(vector<Conflict>({
Conflict("rule1: reduce rule2 / reduce rule1"),
Conflict("rule1: reduce rule1 / reduce rule2")
})));
describe("when one action has higher precedence", [&]() {
ParseAction left = ParseAction::Reduce(sym2, 1, 0);
ParseAction right = ParseAction::Reduce(sym2, 1, 3);
it("favors that action", [&]() {
AssertThat(manager->resolve_parse_action(sym1, left, right), IsTrue());
AssertThat(manager->resolve_parse_action(sym1, right, left), IsFalse());
});
it("does not record a conflict", [&]() {
manager->resolve_parse_action(sym1, left, right);
manager->resolve_parse_action(sym1, right, left);
AssertThat(manager->conflicts(), IsEmpty());
});
});
describe("when the actions have the same precedence", [&]() {
ParseAction left = ParseAction::Reduce(sym1, 1, 0);
ParseAction right = ParseAction::Reduce(sym2, 1, 0);
it("favors the symbol listed earlier in the grammar", [&]() {
AssertThat(manager->resolve_parse_action(sym1, right, left), IsTrue());
AssertThat(manager->resolve_parse_action(sym1, left, right), IsFalse());
});
it("favors the symbol listed earlier in the grammar", [&]() {
should_update = manager->resolve_parse_action(sym1, ParseAction::Reduce(sym2, 1), ParseAction::Reduce(sym1, 1));
AssertThat(should_update, IsTrue());
should_update = manager->resolve_parse_action(sym1, ParseAction::Reduce(sym1, 1), ParseAction::Reduce(sym2, 1));
AssertThat(should_update, IsFalse());
it("records a conflict", [&]() {
manager->resolve_parse_action(sym1, left, right);
manager->resolve_parse_action(sym1, right, left);
AssertThat(manager->conflicts(), Equals(vector<Conflict>({
Conflict("rule1: reduce rule2 (precedence 0) / reduce rule1 (precedence 0)"),
Conflict("rule1: reduce rule1 (precedence 0) / reduce rule2 (precedence 0)")
})));
});
});
});
});

14
spec/compiler/build_tables/rule_transitions_spec.cc
View file

@ -1,5 +1,6 @@
#include "compiler_spec_helper.h"
#include "compiler/build_tables/rule_transitions.h"
#include "compiler/rules/metadata.h"
using namespace rules;
using namespace build_tables;
@ -182,6 +183,19 @@ describe("rule transitions", []() {
{ CharacterSet({ 'a' }), rule }
})));
});
it("preserves metadata", [&]() {
map<MetadataKey, int> metadata_value({
{ PRECEDENCE, 5 }
});
rule_ptr rule = make_shared<Metadata>(seq({ sym("x"), sym("y") }), metadata_value);
AssertThat(
sym_transitions(rule),
Equals(rule_map<Symbol>({
{ Symbol("x"), make_shared<Metadata>(sym("y"), metadata_value)},
})));
});
describe("regression tests (somewhat redundant, should maybe be deleted later)", []() {
it("handles sequences that start with repeating characters", [&]() {

4
spec/runtime/languages/arithmetic/errors.txt
View file

@ -12,6 +12,4 @@ x + (y * + z) * 5
---
(expression (sum
(variable)
(product
(grouping (ERROR))
(number))))
(product (group (ERROR)) (number))))

12
spec/runtime/languages/arithmetic/main.txt
View file

@ -30,14 +30,14 @@ x + x
(variable)
(variable)))
====================
parses complex trees
====================
x * y + z / a
===============================================
binds multiplication more tightly than addition
===============================================
a * b + c * d
---
(expression (sum
(product (variable) (variable))
(quotient (variable) (variable))))
(product (variable) (variable))))
============================
parses exponents
@ -50,4 +50,4 @@ x + y * z^(a + b)
(variable)
(exponent
(variable)
(grouping (sum (variable) (variable)))))))
(group (sum (variable) (variable)))))))

2
spec/runtime/node_position_spec.cc
View file

@ -18,8 +18,8 @@ describe("tracking the positions of AST nodes", []() {
it("records the widths and offsets of nodes", [&]() {
ts_document_set_input_string(doc, " [12, 5]");
const ts_tree *tree = ts_document_tree(doc);
const ts_tree *tree = ts_document_tree(doc);
const ts_tree *array = ts_tree_children(tree, NULL)[0];
const ts_tree *number1 = ts_tree_children(array, NULL)[0];
const ts_tree *number2 = ts_tree_children(array, NULL)[1];

30
src/compiler/build_tables/build_tables.cc
View file

@ -2,11 +2,13 @@
#include <string>
#include <utility>
#include <map>
#include <set>
#include <unordered_map>
#include "compiler/prepared_grammar.h"
#include "compiler/rules/built_in_symbols.h"
#include "compiler/rules/metadata.h"
#include "compiler/rules/repeat.h"
#include "compiler/rules/blank.h"
#include "compiler/rules/seq.h"
#include "compiler/build_tables/conflict_manager.h"
#include "compiler/build_tables/item.h"
@ -19,6 +21,7 @@ namespace tree_sitter {
using std::string;
using std::map;
using std::vector;
using std::set;
using std::unordered_map;
using std::make_shared;
using rules::Symbol;
@ -32,17 +35,27 @@ namespace tree_sitter {
unordered_map<const ParseItemSet, ParseStateId> parse_state_ids;
unordered_map<const LexItemSet, LexStateId> lex_state_ids;
set<int> precedence_values_for_item_set(const ParseItemSet &item_set) {
set<int> result;
for (const auto &item : item_set)
if (item.consumed_symbol_count > 0)
result.insert(item.precedence());
return result;
}
void add_shift_actions(const ParseItemSet &item_set, ParseStateId state_id) {
for (auto &transition : sym_transitions(item_set, grammar)) {
const Symbol &symbol = transition.first;
const ParseItemSet &item_set = transition.second;
set<int> precedence_values = precedence_values_for_item_set(item_set);
auto current_actions = parse_table.states[state_id].actions;
auto current_action = current_actions.find(symbol);
if (current_action == current_actions.end() ||
conflict_manager.resolve_parse_action(symbol, current_action->second, ParseAction::Shift(0))) {
conflict_manager.resolve_parse_action(symbol, current_action->second, ParseAction::Shift(0, precedence_values))) {
ParseStateId new_state_id = add_parse_state(item_set);
parse_table.add_action(state_id, symbol, ParseAction::Shift(new_state_id));
parse_table.add_action(state_id, symbol, ParseAction::Shift(new_state_id, precedence_values));
}
}
}
@ -59,7 +72,7 @@ namespace tree_sitter {
void add_token_start(const LexItemSet &item_set, LexStateId state_id) {
for (auto &item : item_set)
if (item.get_metadata(rules::START_TOKEN))
if (item.is_token_start())
lex_table.state(state_id).is_token_start = true;
}
@ -79,12 +92,14 @@ namespace tree_sitter {
if (item.is_done()) {
ParseAction action = (item.lhs == rules::START()) ?
ParseAction::Accept() :
ParseAction::Reduce(item.lhs, item.consumed_symbol_count);
ParseAction::Reduce(item.lhs, item.consumed_symbol_count, item.precedence());
auto current_actions = parse_table.states[state_id].actions;
auto current_action = current_actions.find(item.lookahead_sym);
if (current_action == current_actions.end() ||
conflict_manager.resolve_parse_action(item.lookahead_sym, current_action->second, action))
conflict_manager.resolve_parse_action(item.lookahead_sym, current_action->second, action)) {
parse_table.add_action(state_id, item.lookahead_sym, action);
}
}
}
}
@ -92,7 +107,10 @@ namespace tree_sitter {
rules::rule_ptr after_separators(rules::rule_ptr rule) {
return rules::Seq::Build({
make_shared<rules::Repeat>(CharacterSet({ ' ', '\t', '\n', '\r' }).copy()),
make_shared<rules::Metadata>(rule, map<rules::MetadataKey, int>({ {rules::START_TOKEN, 1} }))
make_shared<rules::Metadata>(make_shared<rules::Blank>(), map<rules::MetadataKey, int>({
{rules::START_TOKEN, 1},
})),
rule
});
}

104
src/compiler/build_tables/conflict_manager.cc
View file

@ -3,35 +3,16 @@
#include <map>
#include <string>
#include <set>
#include "compiler/util/string_helpers.h"
namespace tree_sitter {
namespace build_tables {
using rules::Symbol;
using std::vector;
using std::string;
using std::to_string;
using std::map;
string message_for_action(const ParseAction &action, const map<Symbol, string> &rule_names) {
switch (action.type) {
case ParseActionTypeShift:
return "shift";
case ParseActionTypeReduce: {
auto pair = rule_names.find(action.symbol);
if (pair != rule_names.end())
return "reduce " + pair->second;
else
return "ERROR " + action.symbol.name;
}
case ParseActionTypeAccept:
return "accept";
default:
return "error";
}
}
void ConflictManager::record_conflict(const rules::Symbol &symbol, const ParseAction &left, const ParseAction &right) {
conflicts_.insert(Conflict(rule_names.find(symbol)->second + ": " + message_for_action(left, rule_names) + " / " + message_for_action(right, rule_names)));
}
using std::set;
using std::vector;
ConflictManager::ConflictManager(const PreparedGrammar &parse_grammar,
const PreparedGrammar &lex_grammar,
@ -50,24 +31,42 @@ namespace tree_sitter {
switch (old_action.type) {
case ParseActionTypeError:
return true;
case ParseActionTypeShift:
case ParseActionTypeShift: {
int min_precedence = *old_action.precedence_values.begin();
int max_precedence = *old_action.precedence_values.rbegin();
switch (new_action.type) {
case ParseActionTypeShift:
record_conflict(symbol, old_action, new_action);
return false;
case ParseActionTypeReduce:
record_conflict(symbol, old_action, new_action);
return false;
case ParseActionTypeReduce: {
int new_precedence = *new_action.precedence_values.rbegin();
if (max_precedence > new_precedence) {
if (min_precedence < new_precedence)
record_conflict(symbol, old_action, new_action);
return false;
} else if (max_precedence < new_precedence) {
return true;
} else {
record_conflict(symbol, old_action, new_action);
return false;
}
}
default:
return false;
}
}
case ParseActionTypeReduce:
switch (new_action.type) {
case ParseActionTypeReduce: {
record_conflict(symbol, old_action, new_action);
size_t old_index = parse_grammar.index_of(old_action.symbol);
size_t new_index = parse_grammar.index_of(new_action.symbol);
return new_index < old_index;
int old_precedence = *old_action.precedence_values.begin();
int new_precedence = *new_action.precedence_values.begin();
if (new_precedence > old_precedence) {
return true;
} else if (new_precedence < old_precedence) {
return false;
} else {
record_conflict(symbol, old_action, new_action);
size_t old_index = parse_grammar.index_of(old_action.symbol);
size_t new_index = parse_grammar.index_of(new_action.symbol);
return new_index < old_index;
}
}
default:
return false;
@ -98,5 +97,42 @@ namespace tree_sitter {
result.insert(result.end(), conflicts_.begin(), conflicts_.end());
return result;
}
string precedence_string(const ParseAction &action) {
string precedences = "(precedence ";
bool started = false;
for (auto value : action.precedence_values) {
if (started) precedences += ", ";
started = true;
precedences += to_string(value);
}
return precedences + ")";
}
string message_for_action(const ParseAction &action, const map<Symbol, string> &rule_names) {
switch (action.type) {
case ParseActionTypeShift:
return "shift " + precedence_string(action);
case ParseActionTypeReduce: {
auto pair = rule_names.find(action.symbol);
if (pair == rule_names.end())
return "ERROR " + action.symbol.name;
else
return "reduce " + pair->second + " " + precedence_string(action);
}
case ParseActionTypeAccept:
return "accept";
default:
return "error";
}
}
void ConflictManager::record_conflict(const rules::Symbol &symbol,
const ParseAction &left,
const ParseAction &right) {
conflicts_.insert(Conflict(rule_names.find(symbol)->second + ": " +
message_for_action(left, rule_names) + " / " +
message_for_action(right, rule_names)));
}
}
}

3
src/compiler/build_tables/get_metadata.cc
View file

@ -10,8 +10,7 @@ namespace tree_sitter {
class GetMetadata : public rules::RuleFn<int> {
rules::MetadataKey metadata_key;
public:
GetMetadata(rules::MetadataKey key) :
metadata_key(key) {}
GetMetadata(rules::MetadataKey key) : metadata_key(key) {}
int apply_to(const rules::Choice *rule) {
return apply(rule->left) || apply(rule->right);

12
src/compiler/build_tables/item.cc
View file

@ -20,10 +20,6 @@ namespace tree_sitter {
return rule_can_be_blank(rule);
}
int Item::get_metadata(rules::MetadataKey key) const {
return build_tables::get_metadata(rule, key);
}
ostream& operator<<(ostream &stream, const LexItem &item) {
return stream <<
string("#<item ") <<
@ -53,6 +49,10 @@ namespace tree_sitter {
bool rules_eq = (*other.rule == *rule);
return lhs_eq && rules_eq;
}
bool LexItem::is_token_start() const {
return get_metadata(rule, rules::START_TOKEN) != 0;
}
ParseItem::ParseItem(const Symbol &lhs,
const rule_ptr rule,
@ -69,6 +69,10 @@ namespace tree_sitter {
bool lookaheads_eq = other.lookahead_sym == lookahead_sym;
return lhs_eq && rules_eq && consumed_sym_counts_eq && lookaheads_eq;
}
int ParseItem::precedence() const {
return get_metadata(rule, rules::PRECEDENCE);
}
}
}

3
src/compiler/build_tables/item.h
View file

@ -15,7 +15,6 @@ namespace tree_sitter {
public:
Item(const rules::Symbol &lhs, rules::rule_ptr rule);
bool is_done() const;
int get_metadata(rules::MetadataKey) const;
rules::Symbol lhs;
rules::rule_ptr rule;
@ -25,6 +24,7 @@ namespace tree_sitter {
public:
LexItem(const rules::Symbol &lhs, rules::rule_ptr rule);
bool operator==(const LexItem &other) const;
bool is_token_start() const;
};
class ParseItem : public Item {
@ -34,6 +34,7 @@ namespace tree_sitter {
const size_t consumed_symbol_count,
const rules::Symbol &lookahead_sym);
bool operator==(const ParseItem &other) const;
int precedence() const;
const size_t consumed_symbol_count;
const rules::Symbol lookahead_sym;

5
src/compiler/build_tables/rule_transitions.cc
View file

@ -19,6 +19,7 @@ namespace tree_sitter {
using rules::rule_ptr;
using rules::Symbol;
using rules::CharacterSet;
using rules::Metadata;
namespace build_tables {
template<typename T>
@ -92,7 +93,9 @@ namespace tree_sitter {
}
map<T, rule_ptr> apply_to(const rules::Metadata *rule) {
return this->apply(rule->rule);
return map_transitions(this->apply(rule->rule), [&](const rule_ptr &to_rule) {
return make_shared<Metadata>(to_rule, rule->value);
});
}
map<T, rule_ptr> apply_to(const rules::String *rule) {

5
src/compiler/generate_code/c_code.cc
View file

@ -4,7 +4,7 @@
#include <utility>
#include <vector>
#include "compiler/generate_code/c_code.h"
#include "compiler/generate_code/helpers.h"
#include "compiler/util/string_helpers.h"
#include "compiler/rules/built_in_symbols.h"
namespace tree_sitter {
@ -14,6 +14,9 @@ namespace tree_sitter {
using std::vector;
using std::set;
using std::pair;
using util::join;
using util::indent;
using util::character_code;
namespace generate_code {
string _switch(string condition, string body) {

49
src/compiler/generate_code/helpers.cc
View file

@ -1,49 +0,0 @@
#include "compiler/generate_code/helpers.h"
#include "compiler/util/string_helpers.h"
namespace tree_sitter {
using std::string;
using std::vector;
namespace generate_code {
string join(vector<string> lines, string separator) {
string result;
bool started = false;
for (auto line : lines) {
if (started) result += separator;
started = true;
result += line;
}
return result;
}
string join(vector<string> lines) {
return join(lines, "\n");
}
string indent(string input) {
string tab = " ";
util::str_replace(&input, "\n", "\n" + tab);
return tab + input;
}
string character_code(char character) {
switch (character) {
case '\0':
return "\\0";
case '"':
return "\\\"";
case '\n':
return "\\n";
case '\r':
return "\\r";
case '\t':
return "\\t";
case '\\':
return "\\\\";
default:
return string() + character;
}
}
}
}

16
src/compiler/generate_code/helpers.h
View file

@ -1,16 +0,0 @@
#ifndef COMPILER_GENERATE_CODE_HELPERS_H_
#define COMPILER_GENERATE_CODE_HELPERS_H_
#include <string>
#include <vector>
namespace tree_sitter {
namespace generate_code {
std::string indent(std::string input);
std::string join(std::vector<std::string> lines, std::string separator);
std::string join(std::vector<std::string> lines);
std::string character_code(char character);
}
}
#endif // COMPILER_GENERATE_CODE_HELPERS_H_

20
src/compiler/parse_table.cc
View file

@ -12,11 +12,13 @@ namespace tree_sitter {
ParseAction::ParseAction(ParseActionType type,
size_t state_index,
Symbol symbol,
size_t consumed_symbol_count) :
size_t consumed_symbol_count,
set<int> precedence_values) :
type(type),
symbol(symbol),
state_index(state_index),
consumed_symbol_count(consumed_symbol_count) {}
consumed_symbol_count(consumed_symbol_count),
precedence_values(precedence_values) {}
ParseAction::ParseAction() :
type(ParseActionTypeError),
@ -25,19 +27,19 @@ namespace tree_sitter {
consumed_symbol_count(0) {}
ParseAction ParseAction::Error() {
return ParseAction(ParseActionTypeError, -1, Symbol(""), {});
return ParseAction(ParseActionTypeError, -1, Symbol(""), 0, { 0 });
}
ParseAction ParseAction::Accept() {
return ParseAction(ParseActionTypeAccept, -1, Symbol(""), {});
return ParseAction(ParseActionTypeAccept, -1, Symbol(""), 0, { 0 });
}
ParseAction ParseAction::Shift(size_t state_index) {
return ParseAction(ParseActionTypeShift, state_index, Symbol(""), {});
ParseAction ParseAction::Shift(size_t state_index, set<int> precedence_values) {
return ParseAction(ParseActionTypeShift, state_index, Symbol(""), 0, precedence_values);
}
ParseAction ParseAction::Reduce(Symbol symbol, size_t consumed_symbol_count) {
return ParseAction(ParseActionTypeReduce, -1, symbol, consumed_symbol_count);
ParseAction ParseAction::Reduce(Symbol symbol, size_t consumed_symbol_count, int precedence) {
return ParseAction(ParseActionTypeReduce, -1, symbol, consumed_symbol_count, { precedence });
}
bool ParseAction::operator==(const ParseAction &other) const {
@ -90,6 +92,6 @@ namespace tree_sitter {
void ParseTable::add_action(ParseStateId id, Symbol symbol, ParseAction action) {
symbols.insert(symbol);
states[id].actions.insert({ symbol, action });
states[id].actions[symbol] = action;
}
}

8
src/compiler/parse_table.h
View file

@ -20,19 +20,21 @@ namespace tree_sitter {
ParseAction(ParseActionType type,
size_t state_index,
rules::Symbol symbol,
size_t consumed_symbol_count);
size_t consumed_symbol_count,
std::set<int> precedence_values);
public:
ParseAction();
static ParseAction Accept();
static ParseAction Error();
static ParseAction Shift(size_t state_index);
static ParseAction Reduce(rules::Symbol symbol, size_t consumed_symbol_count);
static ParseAction Shift(size_t state_index, std::set<int> precedence_values);
static ParseAction Reduce(rules::Symbol symbol, size_t consumed_symbol_count, int precedence);
bool operator==(const ParseAction &action) const;
ParseActionType type;
rules::Symbol symbol;
size_t state_index;
size_t consumed_symbol_count;
std::set<int> precedence_values;
};
std::ostream& operator<<(std::ostream &stream, const ParseAction &item);

9
src/compiler/rules/rules.cc
View file

@ -1,4 +1,5 @@
#include <vector>
#include <map>
#include <set>
#include <string>
#include "tree_sitter/compiler.h"
@ -8,6 +9,7 @@
#include "compiler/rules/choice.h"
#include "compiler/rules/seq.h"
#include "compiler/rules/string.h"
#include "compiler/rules/metadata.h"
#include "compiler/rules/pattern.h"
#include "compiler/rules/character_set.h"
#include "compiler/rules/repeat.h"
@ -18,6 +20,7 @@ namespace tree_sitter {
using std::string;
using std::set;
using std::vector;
using std::map;
namespace rules {
rule_ptr blank() {
@ -51,5 +54,11 @@ namespace tree_sitter {
rule_ptr err(const rule_ptr &rule) {
return choice({ rule, ERROR().copy() });
}
rule_ptr prec(int precedence, rule_ptr rule) {
return std::make_shared<Metadata>(rule, map<MetadataKey, int>({
{ PRECEDENCE, precedence }
}));
}
}
}

43
src/compiler/util/string_helpers.cc
View file

@ -1,7 +1,10 @@
#include "compiler/util/string_helpers.h"
#include <vector>
namespace tree_sitter {
using std::string;
using std::vector;
using std::set;
namespace util {
void str_replace(string *input, const string &search, const string &replace) {
@ -20,5 +23,45 @@ namespace tree_sitter {
str_replace(&input, "\n", "\\n");
return input;
}
string join(vector<string> lines, string separator) {
string result;
bool started = false;
for (auto line : lines) {
if (started) result += separator;
started = true;
result += line;
}
return result;
}
string join(vector<string> lines) {
return join(lines, "\n");
}
string indent(string input) {
string tab = " ";
util::str_replace(&input, "\n", "\n" + tab);
return tab + input;
}
string character_code(char character) {
switch (character) {
case '\0':
return "\\0";
case '"':
return "\\\"";
case '\n':
return "\\n";
case '\r':
return "\\r";
case '\t':
return "\\t";
case '\\':
return "\\\\";
default:
return string() + character;
}
}
}
}

6
src/compiler/util/string_helpers.h
View file

@ -2,11 +2,17 @@
#define COMPILER_UTIL_STRING_HELPERS_H_
#include <string>
#include <vector>
#include <set>
namespace tree_sitter {
namespace util {
void str_replace(std::string *input, const std::string &search, const std::string &replace);
std::string escape_string(std::string input);
std::string indent(std::string input);
std::string join(std::vector<std::string> lines, std::string separator);
std::string join(std::vector<std::string> lines);
std::string character_code(char character);
}
}