traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Replace prec rule w/ left_assoc and right_assoc

Browse source 
Consider shift/reduce conflicts to be compilation errors unless
they are resolved by a specified associativity.
This commit is contained in:
Max Brunsfeld 2015-03-16 23:12:08 -07:00
parent 86bd6eaa75
commit 80ec303b10
19 changed files with 27040 additions and 25946 deletions
Download patch file Download diff file Expand all files Collapse all files

3
include/tree_sitter/compiler.h
View file

@ -24,7 +24,8 @@ rule_ptr str(const std::string &);
rule_ptr keyword(const std::string &);
rule_ptr keypattern(const std::string &);
rule_ptr err(const rule_ptr &);
rule_ptr prec(int precedence, const rule_ptr &);
rule_ptr left_assoc(int precedence, const rule_ptr &);
rule_ptr right_assoc(int precedence, const rule_ptr &);
rule_ptr token(const rule_ptr &rule);
std::ostream &operator<<(std::ostream &stream, const rules::rule_ptr &rule);

2
spec/compiler/build_tables/build_parse_table_spec.cc
View file

@ -60,7 +60,7 @@ describe("build_parse_table", []() {
auto result = build_parse_table(parse_grammar, lex_grammar);
AssertThat(result.first.states[2].actions, Equals(map<Symbol, ParseAction>({
{ END_OF_INPUT(), ParseAction::Reduce(Symbol(0), 1, 0, 0) },
{ END_OF_INPUT(), ParseAction::Reduce(Symbol(0), 1, 0, AssociativityLeft, 0) },
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },

92
spec/compiler/build_tables/parse_conflict_manager_spec.cc
View file

@ -39,7 +39,17 @@ describe("ParseConflictManager", []() {
describe(".get_production_id", [&]() {
it("returns different IDs for different productions", [&]() {
int id1 = conflict_manager->get_production_id(vector<Symbol>({ Symbol(1), Symbol(2) }));
AssertThat(id1, Equals(0));
int id2 = conflict_manager->get_production_id(vector<Symbol>({ Symbol(1), Symbol(2), Symbol(3) }));
AssertThat(id2, Equals(1));
int id3 = conflict_manager->get_production_id(vector<Symbol>({ Symbol(1) }));
AssertThat(id3, Equals(2));
int id4 = conflict_manager->get_production_id(vector<Symbol>({ Symbol(1), Symbol(2) }));
AssertThat(id4, Equals(id1));
});
});
@ -62,7 +72,7 @@ describe("ParseConflictManager", []() {
describe("shift/reduce conflicts", [&]() {
describe("when the shift has higher precedence", [&]() {
ParseAction shift = ParseAction::Shift(2, { 3 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 1, 0);
ParseAction reduce = ParseAction::Reduce(sym2, 1, 1, AssociativityLeft, 0);
it("favors the shift and reports the conflict as resolved", [&]() {
result = conflict_manager->resolve(shift, reduce, sym1);
@ -76,8 +86,10 @@ describe("ParseConflictManager", []() {
});
describe("when the reduce has higher precedence", [&]() {
ParseAction shift = ParseAction::Shift(2, { 1 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 3, 0);
ParseAction shift = ParseAction::Shift(2, {
{1, AssociativityLeft}
});
ParseAction reduce = ParseAction::Reduce(sym2, 1, 3, AssociativityLeft, 0);
it("favors the reduce and reports the conflict as resolved", [&]() {
result = conflict_manager->resolve(shift, reduce, sym1);
@ -90,11 +102,25 @@ describe("ParseConflictManager", []() {
});
});
describe("when the precedences are equal", [&]() {
describe("when the precedences are equal and the reduce's rule is left associative", [&]() {
ParseAction shift = ParseAction::Shift(2, { 0 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 0, 0);
ParseAction reduce = ParseAction::Reduce(sym2, 1, 0, AssociativityLeft, 0);
it("favors the reduce and reports the conflict as resolved", [&]() {
result = conflict_manager->resolve(reduce, shift, sym1);
AssertThat(get<0>(result), IsTrue());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(shift, reduce, sym1);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
});
});
describe("when the precedences are equal and the reduce's rule is right-associative", [&]() {
ParseAction shift = ParseAction::Shift(2, { 0 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 0, AssociativityRight, 0);
// TODO: Add associativity annotations. These should be errors.
it("favors the shift, and reports the conflict as resolved", [&]() {
result = conflict_manager->resolve(reduce, shift, sym1);
AssertThat(get<0>(result), IsFalse());
@ -106,27 +132,59 @@ describe("ParseConflictManager", []() {
});
});
describe("when the precedences are equal and the reduce's rule has no associativity", [&]() {
it("reports an unresolved conflict", [&]() {
ParseAction shift = ParseAction::Shift(2, { 0 });
ParseAction reduce = ParseAction::Reduce(Symbol(2), 1, 0, AssociativityUnspecified, 0);
reduce.production_id = conflict_manager->get_production_id(vector<Symbol>({
Symbol(3),
Symbol(4),
}));
result = conflict_manager->resolve(reduce, shift, lookahead_sym);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeError));
AssertThat(get<2>(result), Equals(
"Lookahead: lookahead_token\n"
"Possible Actions:\n"
"* Shift\n"
"* Reduce other_rule1 other_rule2 -> reduced_rule"
));
result = conflict_manager->resolve(shift, reduce, lookahead_sym);
AssertThat(get<0>(result), IsTrue());
AssertThat(get<2>(result), Equals(
"Lookahead: lookahead_token\n"
"Possible Actions:\n"
"* Shift\n"
"* Reduce other_rule1 other_rule2 -> reduced_rule"
));
});
});
describe("when the shift has conflicting precedences compared to the reduce", [&]() {
ParseAction shift = ParseAction::Shift(2, { 0, 1, 3 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 2, 0);
ParseAction reduce = ParseAction::Reduce(sym2, 1, 2, AssociativityLeft, 0);
// TODO: Add associativity annotations. These should be errors.
it("favors the shift, and reports the conflict as resolved", [&]() {
it("returns false and reports an unresolved conflict", [&]() {
reduce.production_id = conflict_manager->get_production_id({ Symbol(1) });
result = conflict_manager->resolve(reduce, shift, sym2);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
AssertThat(get<1>(result), Equals(ConflictTypeError));
result = conflict_manager->resolve(shift, reduce, sym1);
AssertThat(get<0>(result), IsTrue());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
AssertThat(get<1>(result), Equals(ConflictTypeError));
});
});
});
describe("reduce/reduce conflicts", [&]() {
describe("when one action has higher precedence", [&]() {
ParseAction left = ParseAction::Reduce(sym2, 1, 0, 0);
ParseAction right = ParseAction::Reduce(sym2, 1, 3, 0);
ParseAction left = ParseAction::Reduce(sym2, 1, 0, AssociativityLeft, 0);
ParseAction right = ParseAction::Reduce(sym2, 1, 3, AssociativityLeft, 0);
it("favors that action", [&]() {
result = conflict_manager->resolve(left, right, sym1);
@ -141,8 +199,8 @@ describe("ParseConflictManager", []() {
describe("when the actions have the same precedence", [&]() {
it("returns false and reports a conflict", [&]() {
ParseAction left = ParseAction::Reduce(Symbol(2), 1, 0, 0);
ParseAction right = ParseAction::Reduce(Symbol(3), 1, 0, 0);
ParseAction left = ParseAction::Reduce(Symbol(2), 1, 0, AssociativityLeft, 0);
ParseAction right = ParseAction::Reduce(Symbol(3), 1, 0, AssociativityLeft, 0);
left.production_id = conflict_manager->get_production_id(vector<Symbol>({
Symbol(3),
@ -160,7 +218,7 @@ describe("ParseConflictManager", []() {
"Lookahead: lookahead_token\n"
"Possible Actions:\n"
"* Reduce other_rule1 other_rule2 -> reduced_rule\n"
"* Reduce other_rule2 -> other_rule1\n"
"* Reduce other_rule2 -> other_rule1"
));
result = conflict_manager->resolve(left, right, lookahead_sym);
@ -170,7 +228,7 @@ describe("ParseConflictManager", []() {
"Lookahead: lookahead_token\n"
"Possible Actions:\n"
"* Reduce other_rule2 -> other_rule1\n"
"* Reduce other_rule1 other_rule2 -> reduced_rule\n"
"* Reduce other_rule1 other_rule2 -> reduced_rule"
));
});
});

4
spec/fixtures/grammars/golang.cc vendored
View file

@ -146,7 +146,7 @@ extern const Grammar golang = Grammar({
infix_op(">=", "expression", 3),
infix_op(">", "expression", 3),
prefix_op("!", "expression", 4) }) },
{ "_func_signature", seq({
{ "_func_signature", left_assoc(10, seq({
in_parens(comma_sep(seq({
comma_sep1(sym("var_name")),
sym("type_expression") }))),
@ -155,7 +155,7 @@ extern const Grammar golang = Grammar({
comma_sep1(seq({ sym("var_name"), sym("type_name") })),
comma_sep1(sym("type_name")) })),
sym("type_name"),
blank() }) }) },
blank() }) })) },
{ "_line_break", str("\n") },

6
spec/fixtures/grammars/helpers.cc vendored
View file

@ -29,20 +29,20 @@ rule_ptr in_brackets(rule_ptr rule) {
}
rule_ptr infix_op(std::string op, std::string rule_name, int precedence) {
return prec(precedence, seq({
return left_assoc(precedence, seq({
sym(rule_name),
keyword(op),
sym(rule_name) }));
}
rule_ptr prefix_op(std::string op, std::string rule_name, int precedence) {
return prec(precedence, seq({
return right_assoc(precedence, seq({
keyword(op),
sym(rule_name) }));
}
rule_ptr postfix_op(std::string op, std::string rule_name, int precedence) {
return prec(precedence, seq({
return left_assoc(precedence, seq({
sym(rule_name),
keyword(op) }));
}

54
spec/fixtures/grammars/javascript.cc vendored
View file

@ -27,7 +27,8 @@ enum {
PREC_SIGN,
PREC_INC,
PREC_CALL,
PREC_CONSTRUCT,
PREC_SHORT_NEW,
PREC_FULL_NEW,
PREC_MEMBER,
};
@ -64,16 +65,16 @@ extern const Grammar javascript = Grammar({
sym("identifier"),
sym("var_assignment") }))) })) },
{ "statement_block", prec(PREC_BLOCK,
{ "statement_block", left_assoc(PREC_BLOCK,
in_braces(err(repeat(sym("statement"))))) },
{ "if_statement", seq({
{ "if_statement", right_assoc(0, seq({
keyword("if"),
sym("_paren_expression"),
sym("statement"),
optional(seq({
keyword("else"),
sym("statement") })) }) },
sym("statement") })) })) },
{ "switch_statement", seq({
keyword("switch"),
@ -98,9 +99,10 @@ extern const Grammar javascript = Grammar({
keyword("for"),
str("("),
optional(keyword("var")),
sym("identifier"),
keyword("in"),
sym("expression"),
left_assoc(PREC_REL, seq({
sym("identifier"),
keyword("in"),
sym("expression") })),
str(")"),
sym("statement") }) },
@ -185,7 +187,7 @@ extern const Grammar javascript = Grammar({
sym("ternary"),
sym("bool_op"),
sym("math_op"),
sym("comma_op"),
// sym("comma_op"),
// sym("bitwise_op"),
@ -213,32 +215,35 @@ extern const Grammar javascript = Grammar({
str(")"),
sym("statement_block") }) },
{ "function_call", prec(PREC_CALL, seq({
{ "function_call", left_assoc(PREC_CALL, seq({
sym("expression"),
str("("),
optional(err(sym("arguments"))),
str(")") })) },
{ "constructor_call", prec(PREC_CONSTRUCT, seq({
keyword("new"),
sym("expression"),
optional(seq({
str("("),
err(optional(sym("arguments"))),
str(")") })) })) },
{ "constructor_call", choice({
left_assoc(PREC_SHORT_NEW, seq({
keyword("new"),
sym("expression") })),
left_assoc(PREC_FULL_NEW, seq({
keyword("new"),
sym("expression"),
str("("),
err(optional(sym("arguments"))),
str(")") })) }) },
{ "member_access", prec(PREC_MEMBER, seq({
{ "member_access", left_assoc(PREC_MEMBER, seq({
sym("expression"),
str("."),
sym("identifier") })) },
{ "subscript_access", prec(PREC_MEMBER, seq({
{ "subscript_access", left_assoc(PREC_MEMBER, seq({
sym("expression"),
str("["),
err(sym("expression")),
str("]") })) },
{ "assignment", prec(PREC_ASSIGN, seq({
{ "assignment", right_assoc(PREC_ASSIGN, seq({
choice({
sym("identifier"),
sym("member_access"),
@ -246,7 +251,7 @@ extern const Grammar javascript = Grammar({
str("="),
sym("expression") })) },
{ "math_assignment", prec(PREC_ASSIGN, seq({
{ "math_assignment", right_assoc(PREC_ASSIGN, seq({
choice({
sym("identifier"),
sym("member_access"),
@ -254,7 +259,7 @@ extern const Grammar javascript = Grammar({
choice({ str("+="), str("-="), str("*="), str("/=") }),
sym("expression") })) },
{ "ternary", prec(PREC_TERNARY, seq({
{ "ternary", right_assoc(PREC_TERNARY, seq({
sym("expression"),
str("?"),
sym("expression"),
@ -299,7 +304,10 @@ extern const Grammar javascript = Grammar({
infix_op(">", "expression", PREC_REL) }) },
{ "type_op", choice({
infix_op("in", "expression", PREC_REL),
left_assoc(PREC_REL, seq({
choice({ sym("expression"), sym("identifier") }),
keyword("in"),
sym("expression") })),
infix_op("instanceof", "expression", PREC_REL),
prefix_op("typeof", "expression", PREC_TYPE) }) },
@ -336,7 +344,7 @@ extern const Grammar javascript = Grammar({
{ "formal_parameters", comma_sep1(sym("identifier")) },
{ "arguments", comma_sep1(err(sym("expression"))) },
{ "arguments", left_assoc(-5, comma_sep1(err(sym("expression")))) },
{ "pair", seq({
choice({ sym("string"), sym("identifier") }),

36
spec/fixtures/parsers/arithmetic.c vendored
View file

@ -452,15 +452,15 @@ static const TSParseAction ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[aux_sym_STR_DASH] = REDUCE_FRAGILE(sym_exponent, 3),
[aux_sym_STR_STAR] = REDUCE_FRAGILE(sym_exponent, 3),
[aux_sym_STR_SLASH] = REDUCE_FRAGILE(sym_exponent, 3),
[aux_sym_STR_CARET] = SHIFT(15),
[aux_sym_STR_CARET] = REDUCE_FRAGILE(sym_exponent, 3),
[aux_sym_STR_RPAREN] = REDUCE_FRAGILE(sym_exponent, 3),
},
[17] = {
[sym_comment] = SHIFT_EXTRA(),
[aux_sym_STR_PLUS] = REDUCE_FRAGILE(sym_quotient, 3),
[aux_sym_STR_DASH] = REDUCE_FRAGILE(sym_quotient, 3),
[aux_sym_STR_STAR] = SHIFT(13),
[aux_sym_STR_SLASH] = SHIFT(14),
[aux_sym_STR_STAR] = REDUCE_FRAGILE(sym_quotient, 3),
[aux_sym_STR_SLASH] = REDUCE_FRAGILE(sym_quotient, 3),
[aux_sym_STR_CARET] = SHIFT(15),
[aux_sym_STR_RPAREN] = REDUCE_FRAGILE(sym_quotient, 3),
},
@ -468,15 +468,15 @@ static const TSParseAction ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[sym_comment] = SHIFT_EXTRA(),
[aux_sym_STR_PLUS] = REDUCE_FRAGILE(sym_product, 3),
[aux_sym_STR_DASH] = REDUCE_FRAGILE(sym_product, 3),
[aux_sym_STR_STAR] = SHIFT(13),
[aux_sym_STR_SLASH] = SHIFT(14),
[aux_sym_STR_STAR] = REDUCE_FRAGILE(sym_product, 3),
[aux_sym_STR_SLASH] = REDUCE_FRAGILE(sym_product, 3),
[aux_sym_STR_CARET] = SHIFT(15),
[aux_sym_STR_RPAREN] = REDUCE_FRAGILE(sym_product, 3),
},
[19] = {
[sym_comment] = SHIFT_EXTRA(),
[aux_sym_STR_PLUS] = SHIFT(11),
[aux_sym_STR_DASH] = SHIFT(12),
[aux_sym_STR_PLUS] = REDUCE_FRAGILE(sym_difference, 3),
[aux_sym_STR_DASH] = REDUCE_FRAGILE(sym_difference, 3),
[aux_sym_STR_STAR] = SHIFT(13),
[aux_sym_STR_SLASH] = SHIFT(14),
[aux_sym_STR_CARET] = SHIFT(15),
@ -484,8 +484,8 @@ static const TSParseAction ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
},
[20] = {
[sym_comment] = SHIFT_EXTRA(),
[aux_sym_STR_PLUS] = SHIFT(11),
[aux_sym_STR_DASH] = SHIFT(12),
[aux_sym_STR_PLUS] = REDUCE_FRAGILE(sym_sum, 3),
[aux_sym_STR_DASH] = REDUCE_FRAGILE(sym_sum, 3),
[aux_sym_STR_STAR] = SHIFT(13),
[aux_sym_STR_SLASH] = SHIFT(14),
[aux_sym_STR_CARET] = SHIFT(15),
@ -572,15 +572,15 @@ static const TSParseAction ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[aux_sym_STR_DASH] = REDUCE_FRAGILE(sym_exponent, 3),
[aux_sym_STR_STAR] = REDUCE_FRAGILE(sym_exponent, 3),
[aux_sym_STR_SLASH] = REDUCE_FRAGILE(sym_exponent, 3),
[aux_sym_STR_CARET] = SHIFT(26),
[aux_sym_STR_CARET] = REDUCE_FRAGILE(sym_exponent, 3),
},
[28] = {
[ts_builtin_sym_end] = REDUCE_FRAGILE(sym_quotient, 3),
[sym_comment] = SHIFT_EXTRA(),
[aux_sym_STR_PLUS] = REDUCE_FRAGILE(sym_quotient, 3),
[aux_sym_STR_DASH] = REDUCE_FRAGILE(sym_quotient, 3),
[aux_sym_STR_STAR] = SHIFT(24),
[aux_sym_STR_SLASH] = SHIFT(25),
[aux_sym_STR_STAR] = REDUCE_FRAGILE(sym_quotient, 3),
[aux_sym_STR_SLASH] = REDUCE_FRAGILE(sym_quotient, 3),
[aux_sym_STR_CARET] = SHIFT(26),
},
[29] = {
@ -588,15 +588,15 @@ static const TSParseAction ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[sym_comment] = SHIFT_EXTRA(),
[aux_sym_STR_PLUS] = REDUCE_FRAGILE(sym_product, 3),
[aux_sym_STR_DASH] = REDUCE_FRAGILE(sym_product, 3),
[aux_sym_STR_STAR] = SHIFT(24),
[aux_sym_STR_SLASH] = SHIFT(25),
[aux_sym_STR_STAR] = REDUCE_FRAGILE(sym_product, 3),
[aux_sym_STR_SLASH] = REDUCE_FRAGILE(sym_product, 3),
[aux_sym_STR_CARET] = SHIFT(26),
},
[30] = {
[ts_builtin_sym_end] = REDUCE_FRAGILE(sym_difference, 3),
[sym_comment] = SHIFT_EXTRA(),
[aux_sym_STR_PLUS] = SHIFT(22),
[aux_sym_STR_DASH] = SHIFT(23),
[aux_sym_STR_PLUS] = REDUCE_FRAGILE(sym_difference, 3),
[aux_sym_STR_DASH] = REDUCE_FRAGILE(sym_difference, 3),
[aux_sym_STR_STAR] = SHIFT(24),
[aux_sym_STR_SLASH] = SHIFT(25),
[aux_sym_STR_CARET] = SHIFT(26),
@ -604,8 +604,8 @@ static const TSParseAction ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[31] = {
[ts_builtin_sym_end] = REDUCE_FRAGILE(sym_sum, 3),
[sym_comment] = SHIFT_EXTRA(),
[aux_sym_STR_PLUS] = SHIFT(22),
[aux_sym_STR_DASH] = SHIFT(23),
[aux_sym_STR_PLUS] = REDUCE_FRAGILE(sym_sum, 3),
[aux_sym_STR_DASH] = REDUCE_FRAGILE(sym_sum, 3),
[aux_sym_STR_STAR] = SHIFT(24),
[aux_sym_STR_SLASH] = SHIFT(25),
[aux_sym_STR_CARET] = SHIFT(26),

4345
spec/fixtures/parsers/golang.c vendored
View file

File diff suppressed because it is too large Load diff

48293
spec/fixtures/parsers/javascript.c vendored
View file

File diff suppressed because it is too large Load diff

10
spec/runtime/languages/javascript/operators.txt
View file

@ -95,11 +95,13 @@ a++ + b * c - d / e--
---
(expression_statement
(math_op
(math_op
(math_op (identifier))
(math_op
(math_op (identifier) (identifier))
(math_op (identifier) (math_op (identifier))))))
(math_op (identifier))
(math_op (identifier) (identifier)))
(math_op
(identifier)
(math_op (identifier)))))
==========================================
boolean operators

3
src/compiler/build_tables/build_parse_table.cc
View file

@ -111,7 +111,8 @@ class ParseTableBuilder {
(item.lhs == rules::START())
? ParseAction::Accept()
: ParseAction::Reduce(item.lhs, item.consumed_symbols.size(),
item.precedence(), conflict_manager.get_production_id(item.consumed_symbols));
item.precedence(), item.associativity(),
conflict_manager.get_production_id(item.consumed_symbols));
for (const auto &lookahead_sym : lookahead_symbols)
if (should_add_action(state_id, lookahead_sym, action))

60
src/compiler/build_tables/parse_conflict_manager.cc
View file

@ -41,15 +41,19 @@ ParseConflictManager::resolve(const ParseAction &new_action,
return make_tuple(false, ConflictTypeResolved, "");
else if (new_precedence > max_precedence)
return make_tuple(true, ConflictTypeResolved, "");
else {
// TODO: Add associativity annotations. In the event of a precedence
// tie, return ConflictTypeError unless there is an associativity
// annotation to break the tie.
return make_tuple(false, ConflictTypeResolved, "");
else if (min_precedence == max_precedence) {
switch (new_action.associativity) {
case rules::AssociativityLeft:
return make_tuple(true, ConflictTypeResolved, "");
case rules::AssociativityRight:
return make_tuple(false, ConflictTypeResolved, "");
default:
return make_tuple(false, ConflictTypeError, conflict_description(new_action, old_action, symbol));
}
} else {
return make_tuple(false, ConflictTypeError, conflict_description(new_action, old_action, symbol));
}
}
break;
case ParseActionTypeReduce:
if (new_action.type == ParseActionTypeReduce) {
@ -60,20 +64,13 @@ ParseConflictManager::resolve(const ParseAction &new_action,
} else if (new_precedence < old_precedence) {
return make_tuple(false, ConflictTypeResolved, "");
} else {
string message =
"Lookahead: " + symbol_name(symbol) + "\n" +
"Possible Actions:\n"
"* " + action_description(old_action) + "\n" +
"* " + action_description(new_action) + "\n";
return make_tuple(false, ConflictTypeError, message);
return make_tuple(false, ConflictTypeError, conflict_description(new_action, old_action, symbol));
}
}
default:
break;
return make_tuple(false, ConflictTypeNone, "");
}
return make_tuple(false, ConflictTypeNone, "");
}
size_t ParseConflictManager::get_production_id(const vector<rules::Symbol> &symbols) {
@ -87,6 +84,16 @@ size_t ParseConflictManager::get_production_id(const vector<rules::Symbol> &symb
return iter - begin;
}
string ParseConflictManager::conflict_description(const ParseAction &new_action,
const ParseAction &old_action,
const rules::Symbol &symbol) const {
return
"Lookahead: " + symbol_name(symbol) + "\n" +
"Possible Actions:\n"
"* " + action_description(old_action) + "\n" +
"* " + action_description(new_action);
}
string ParseConflictManager::symbol_name(const rules::Symbol &symbol) const {
if (symbol.is_built_in()) {
if (symbol == rules::ERROR())
@ -102,11 +109,22 @@ string ParseConflictManager::symbol_name(const rules::Symbol &symbol) const {
}
string ParseConflictManager::action_description(const ParseAction &action) const {
string result = "Reduce";
for (const rules::Symbol &symbol : productions[action.production_id])
result += " " + symbol_name(symbol);
result += " -> " + symbol_name(action.symbol);
return result;
switch (action.type) {
case ParseActionTypeReduce: {
string result = "Reduce";
for (const rules::Symbol &symbol : productions[action.production_id])
result += " " + symbol_name(symbol);
result += " -> " + symbol_name(action.symbol);
return result;
}
case ParseActionTypeShift: {
return "Shift";
}
default:
return "";
}
}
} // namespace build_tables

7
src/compiler/build_tables/parse_conflict_manager.h
View file

@ -27,15 +27,14 @@ class ParseConflictManager {
public:
ParseConflictManager(const SyntaxGrammar &, const LexicalGrammar &);
size_t get_production_id(const std::vector<rules::Symbol> &);
std::tuple<bool, ConflictType, std::string>
resolve(const ParseAction &, const ParseAction &, const rules::Symbol &) const;
std::tuple<bool, ConflictType, std::string> resolve(
const ParseAction &, const ParseAction &, const rules::Symbol &) const;
private:
std::string symbol_name(const rules::Symbol &) const;
std::string action_description(const ParseAction &) const;
std::string conflict_description(const ParseAction &, const ParseAction &, const rules::Symbol &) const;
};
} // namespace build_tables

6
src/compiler/build_tables/parse_item.cc
View file

@ -1,4 +1,6 @@
#include "compiler/build_tables/parse_item.h"
#include "compiler/build_tables/get_metadata.h"
#include "compiler/rules/metadata.h"
#include "tree_sitter/compiler.h"
namespace tree_sitter {
@ -18,6 +20,10 @@ bool ParseItem::operator==(const ParseItem &other) const {
(rule == other.rule || rule->operator==(*other.rule));
}
rules::Associativity ParseItem::associativity() const {
return rules::Associativity(get_metadata(rule, rules::ASSOCIATIVITY));
}
ostream &operator<<(ostream &stream, const ParseItem &item) {
return stream << string("(item ") << item.lhs << string(" ") << *item.rule
<< string(")");

2
src/compiler/build_tables/parse_item.h
View file

@ -6,6 +6,7 @@
#include <vector>
#include "compiler/build_tables/item.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/metadata.h"
namespace tree_sitter {
namespace build_tables {
@ -15,6 +16,7 @@ class ParseItem : public Item {
ParseItem(const rules::Symbol &lhs, rules::rule_ptr rule,
const std::vector<rules::Symbol> &consumed_symbols);
bool operator==(const ParseItem &other) const;
rules::Associativity associativity() const;
std::vector<rules::Symbol> consumed_symbols;
};

27
src/compiler/parse_table.cc
View file

@ -13,46 +13,57 @@ using rules::Symbol;
ParseAction::ParseAction(ParseActionType type, ParseStateId state_index,
Symbol symbol, size_t consumed_symbol_count,
set<int> precedence_values,
rules::Associativity associativity,
int production_id)
: type(type),
symbol(symbol),
state_index(state_index),
consumed_symbol_count(consumed_symbol_count),
precedence_values(precedence_values),
associativity(associativity),
production_id(production_id) {}
ParseAction::ParseAction()
: type(ParseActionTypeError),
symbol(Symbol(-1)),
state_index(-1),
consumed_symbol_count(0) {}
consumed_symbol_count(0),
associativity(rules::AssociativityUnspecified) {}
ParseAction ParseAction::Error() {
return ParseAction(ParseActionTypeError, -1, Symbol(-1), 0, { 0 }, 0);
return ParseAction();
}
ParseAction ParseAction::Accept() {
return ParseAction(ParseActionTypeAccept, -1, Symbol(-1), 0, { 0 }, 0);
ParseAction action;
action.type = ParseActionTypeAccept;
return action;
}
ParseAction ParseAction::Shift(ParseStateId state_index,
set<int> precedence_values) {
return ParseAction(ParseActionTypeShift, state_index, Symbol(-1), 0,
precedence_values, 0);
precedence_values, rules::AssociativityUnspecified, -1);
}
ParseAction ParseAction::ShiftExtra() {
return ParseAction(ParseActionTypeShiftExtra, 0, Symbol(-1), 0, { 0 }, 0);
ParseAction action;
action.type = ParseActionTypeShiftExtra;
return action;
}
ParseAction ParseAction::ReduceExtra(Symbol symbol) {
return ParseAction(ParseActionTypeReduceExtra, 0, symbol, 0, { 0 }, 0);
ParseAction action;
action.type = ParseActionTypeReduceExtra;
action.symbol = symbol;
return action;
}
ParseAction ParseAction::Reduce(Symbol symbol, size_t consumed_symbol_count,
int precedence, int production_id) {
int precedence, rules::Associativity associativity,
int production_id) {
return ParseAction(ParseActionTypeReduce, 0, symbol, consumed_symbol_count,
{ precedence }, production_id);
{ precedence }, associativity, production_id);
}
bool ParseAction::operator==(const ParseAction &other) const {

7
src/compiler/parse_table.h
View file

@ -7,6 +7,7 @@
#include <vector>
#include "compiler/lex_table.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/metadata.h"
namespace tree_sitter {
@ -24,7 +25,8 @@ typedef enum {
class ParseAction {
ParseAction(ParseActionType type, ParseStateId state_index,
rules::Symbol symbol, size_t consumed_symbol_count,
std::set<int> precedence_values, int production_id);
std::set<int> precedence_values, rules::Associativity,
int production_id);
public:
ParseAction();
@ -33,7 +35,7 @@ class ParseAction {
static ParseAction Shift(ParseStateId state_index,
std::set<int> precedence_values);
static ParseAction Reduce(rules::Symbol symbol, size_t consumed_symbol_count,
int precedence, int production_id);
int precedence, rules::Associativity, int production_id);
static ParseAction ShiftExtra();
static ParseAction ReduceExtra(rules::Symbol symbol);
bool operator==(const ParseAction &) const;
@ -44,6 +46,7 @@ class ParseAction {
ParseStateId state_index;
size_t consumed_symbol_count;
std::set<int> precedence_values;
rules::Associativity associativity;
int production_id;
};

11
src/compiler/rules/metadata.h
View file

@ -8,12 +8,19 @@
namespace tree_sitter {
namespace rules {
typedef enum {
enum MetadataKey {
START_TOKEN,
PRECEDENCE,
IS_TOKEN,
DESCRIPTION,
} MetadataKey;
ASSOCIATIVITY,
};
enum Associativity {
AssociativityUnspecified,
AssociativityLeft,
AssociativityRight,
};
class Metadata : public Rule {
public:

18
src/compiler/rules/rules.cc
View file

@ -47,19 +47,29 @@ rule_ptr pattern(const string &value) { return make_shared<Pattern>(value); }
rule_ptr str(const string &value) { return make_shared<String>(value); }
rule_ptr keyword(const string &value) {
return token(prec(KEYWORD_PRECEDENCE, str(value)));
return token(left_assoc(KEYWORD_PRECEDENCE, str(value)));
}
rule_ptr keypattern(const string &value) {
return token(prec(KEYWORD_PRECEDENCE, pattern(value)));
return token(left_assoc(KEYWORD_PRECEDENCE, pattern(value)));
}
rule_ptr err(const rule_ptr &rule) {
return choice({ rule, ERROR().copy() });
}
rule_ptr prec(int precedence, const rule_ptr &rule) {
return metadata(rule, { { PRECEDENCE, precedence } });
rule_ptr left_assoc(int precedence, const rule_ptr &rule) {
return metadata(rule, {
{ PRECEDENCE, precedence },
{ ASSOCIATIVITY, AssociativityLeft }
});
}
rule_ptr right_assoc(int precedence, const rule_ptr &rule) {
return metadata(rule, {
{ PRECEDENCE, precedence },
{ ASSOCIATIVITY, AssociativityRight }
});
}
rule_ptr token(const rule_ptr &rule) {