traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Treat parse conflicts as errors in grammar compilation

Browse source 
For now, only reduce/reduce conflicts w/ no tie-breaking precedence
are treated as errors. The rest are dropped, because shift/reduce
conflicts are currently very common because we don't have a way
of specifying associativity along w/ precedence.
This commit is contained in:
Max Brunsfeld 2015-03-15 20:31:41 -07:00
parent 8bd11e1b58
commit 9a198562e0
20 changed files with 1394 additions and 1408 deletions
Download patch file Download diff file Expand all files Collapse all files

37
include/tree_sitter/compiler.h
View file

@ -15,17 +15,17 @@ class Rule;
typedef std::shared_ptr<Rule> rule_ptr;
rule_ptr blank();
rule_ptr choice(const std::vector<rule_ptr> &rules);
rule_ptr repeat(const rule_ptr &content);
rule_ptr seq(const std::vector<rule_ptr> &rules);
rule_ptr sym(const std::string &name);
rule_ptr pattern(const std::string &value);
rule_ptr str(const std::string &value);
rule_ptr keyword(const std::string &value);
rule_ptr keypattern(const std::string &value);
rule_ptr err(const rule_ptr &rule);
rule_ptr prec(int precedence, rule_ptr rule);
rule_ptr token(rule_ptr rule);
rule_ptr choice(const std::vector<rule_ptr> &);
rule_ptr repeat(const rule_ptr &);
rule_ptr seq(const std::vector<rule_ptr> &);
rule_ptr sym(const std::string &);
rule_ptr pattern(const std::string &);
rule_ptr str(const std::string &);
rule_ptr keyword(const std::string &);
rule_ptr keypattern(const std::string &);
rule_ptr err(const rule_ptr &);
rule_ptr prec(int precedence, const rule_ptr &);
rule_ptr token(const rule_ptr &rule);
std::ostream &operator<<(std::ostream &stream, const rules::rule_ptr &rule);
@ -45,17 +45,12 @@ class Grammar {
Grammar &ubiquitous_tokens(const std::set<rules::rule_ptr> &);
};
struct Conflict {
explicit Conflict(std::string description);
std::string description;
bool operator==(const Conflict &other) const;
bool operator<(const Conflict &other) const;
};
enum GrammarErrorType {
GrammarErrorTypeRegex,
GrammarErrorTypeUndefinedSymbol,
GrammarErrorTypeInvalidUbiquitousToken
GrammarErrorTypeInvalidUbiquitousToken,
GrammarErrorTypeLexConflict,
GrammarErrorTypeParseConflict,
};
class GrammarError {
@ -66,11 +61,9 @@ class GrammarError {
std::string message;
};
std::tuple<std::string, std::vector<Conflict>, const GrammarError *> compile(
const Grammar &grammar, std::string name);
std::pair<std::string, const GrammarError *> compile(const Grammar &, std::string);
std::ostream &operator<<(std::ostream &stream, const Grammar &grammar);
std::ostream &operator<<(std::ostream &stream, const Conflict &conflict);
std::ostream &operator<<(std::ostream &stream, const GrammarError *error);
} // namespace tree_sitter

1
project.gyp
View file

@ -25,7 +25,6 @@
'src/compiler/build_tables/rule_can_be_blank.cc',
'src/compiler/build_tables/rule_transitions.cc',
'src/compiler/compile.cc',
'src/compiler/conflict.cc',
'src/compiler/generate_code/c_code.cc',
'src/compiler/grammar.cc',
'src/compiler/lex_table.cc',

36
spec/compiler/build_tables/action_takes_precedence_spec.cc
View file

@ -59,7 +59,7 @@ describe("action_takes_precedence", []() {
});
describe("parsing conflicts", [&]() {
pair<bool, bool> result;
pair<bool, ConflictType> result;
Symbol sym1(0);
Symbol sym2(1);
@ -77,10 +77,10 @@ describe("action_takes_precedence", []() {
it("is not a conflict", [&]() {
result = action_takes_precedence(non_error, error, sym1);
AssertThat(result.second, IsFalse());
AssertThat(result.second, Equals(ConflictTypeNone));
result = action_takes_precedence(error, non_error, sym1);
AssertThat(result.second, IsFalse());
AssertThat(result.second, Equals(ConflictTypeNone));
});
});
@ -91,10 +91,10 @@ describe("action_takes_precedence", []() {
it("is not a conflict", [&]() {
result = action_takes_precedence(shift, reduce, sym1);
AssertThat(result.second, IsFalse());
AssertThat(result.second, Equals(ConflictTypeResolved));
result = action_takes_precedence(reduce, shift, sym1);
AssertThat(result.second, IsFalse());
AssertThat(result.second, Equals(ConflictTypeResolved));
});
it("favors the shift", [&]() {
@ -112,10 +112,10 @@ describe("action_takes_precedence", []() {
it("is not a conflict", [&]() {
result = action_takes_precedence(shift, reduce, sym1);
AssertThat(result.second, IsFalse());
AssertThat(result.second, Equals(ConflictTypeResolved));
result = action_takes_precedence(reduce, shift, sym1);
AssertThat(result.second, IsFalse());
AssertThat(result.second, Equals(ConflictTypeResolved));
});
it("favors the reduce", [&]() {
@ -131,12 +131,13 @@ describe("action_takes_precedence", []() {
ParseAction shift = ParseAction::Shift(2, { 0 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 0, 0);
// TODO: Add associativity annotations. These should be errors.
it("is a conflict", [&]() {
result = action_takes_precedence(reduce, shift, sym1);
AssertThat(result.second, IsTrue());
AssertThat(result.second, Equals(ConflictTypeResolved));
result = action_takes_precedence(shift, reduce, sym1);
AssertThat(result.second, IsTrue());
AssertThat(result.second, Equals(ConflictTypeResolved));
});
it("favors the shift", [&]() {
@ -152,12 +153,13 @@ describe("action_takes_precedence", []() {
ParseAction shift = ParseAction::Shift(2, { 0, 1, 3 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 2, 0);
// TODO: Add associativity annotations. These should be errors.
it("is a conflict", [&]() {
result = action_takes_precedence(reduce, shift, sym1);
AssertThat(result.second, IsTrue());
AssertThat(result.second, Equals(ConflictTypeResolved));
result = action_takes_precedence(shift, reduce, sym1);
AssertThat(result.second, IsTrue());
AssertThat(result.second, Equals(ConflictTypeResolved));
});
it("favors the shift", [&]() {
@ -185,10 +187,10 @@ describe("action_takes_precedence", []() {
it("is not a conflict", [&]() {
result = action_takes_precedence(left, right, sym1);
AssertThat(result.second, IsFalse());
AssertThat(result.second, Equals(ConflictTypeResolved));
result = action_takes_precedence(right, left, sym1);
AssertThat(result.second, IsFalse());
AssertThat(result.second, Equals(ConflictTypeResolved));
});
});
@ -196,9 +198,9 @@ describe("action_takes_precedence", []() {
ParseAction left = ParseAction::Reduce(sym1, 1, 0, 0);
ParseAction right = ParseAction::Reduce(sym2, 1, 0, 0);
it("favors the symbol listed earlier in the grammar", [&]() {
it("returns false", [&]() {
result = action_takes_precedence(left, right, sym1);
AssertThat(result.first, IsTrue());
AssertThat(result.first, IsFalse());
result = action_takes_precedence(right, left, sym1);
AssertThat(result.first, IsFalse());
@ -206,10 +208,10 @@ describe("action_takes_precedence", []() {
it("records a conflict", [&]() {
result = action_takes_precedence(left, right, sym1);
AssertThat(result.second, IsTrue());
AssertThat(result.second, Equals(ConflictTypeError));
result = action_takes_precedence(right, left, sym1);
AssertThat(result.second, IsTrue());
AssertThat(result.second, Equals(ConflictTypeError));
});
});
});

6
spec/compiler/build_tables/build_conflict_spec.cc
View file

@ -9,7 +9,7 @@ using namespace build_tables;
START_TEST
describe("build_conflict", []() {
Conflict conflict("");
string conflict;
SyntaxGrammar parse_grammar({
{ "in_progress_rule1", i_token(0) },
@ -46,7 +46,7 @@ describe("build_conflict", []() {
parse_grammar, lex_grammar
);
AssertThat(conflict.description, Equals(
AssertThat(conflict, Equals(
"lookahead_token: "
"shift ( in_progress_rule1 in_progress_rule2 ) / "
"reduce ( reduced_rule )"));
@ -70,7 +70,7 @@ describe("build_conflict", []() {
parse_grammar, lex_grammar
);
AssertThat(conflict.description, Equals(
AssertThat(conflict, Equals(
"lookahead_token: "
"shift ( in_progress_rule1 in_progress_rule2 ) / "
"reduce ( reduced_rule )"));

8
spec/compiler/compile_examples.cc
View file

@ -25,15 +25,11 @@ describe("compiling the example grammars", []() {
auto compile_grammar = [&](const Grammar &grammar, string language) {
it(("compiles the " + language + " grammar").c_str(), [&]() {
auto result = compile(grammar, language);
string code = get<0>(result);
vector<Conflict> conflicts = get<1>(result);
const GrammarError *error = get<2>(result);
string code = result.first;
const GrammarError *error = result.second;
AssertThat(error, Equals((GrammarError *)nullptr));
// for (const auto &conflict : conflicts)
// std::cout << conflict << std::endl;
ofstream file(example_parser_dir + language + ".c");
file << get<0>(result);
file.close();

2
spec/compiler/compile_spec.cc
View file

@ -13,7 +13,7 @@ describe("Compile", []() {
});
auto result = compile(grammar, "test_grammar");
const GrammarError *error = get<2>(result);
const GrammarError *error = result.second;
AssertThat(error, Equals<const GrammarError *>(nullptr));
});
});

4
spec/fixtures/grammars/javascript.cc vendored
View file

@ -15,6 +15,7 @@ static rule_ptr terminated(rule_ptr rule) {
enum {
PREC_COMMA = -1,
PREC_ASSIGN,
PREC_BLOCK,
PREC_TERNARY,
PREC_OR,
PREC_AND,
@ -63,7 +64,8 @@ extern const Grammar javascript = Grammar({
sym("identifier"),
sym("var_assignment") }))) })) },
{ "statement_block", in_braces(err(repeat(sym("statement")))) },
{ "statement_block", prec(PREC_BLOCK,
in_braces(err(repeat(sym("statement"))))) },
{ "if_statement", seq({
keyword("if"),

126
spec/fixtures/parsers/golang.c vendored
View file

@ -4670,22 +4670,22 @@ static const TSParseAction ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[aux_sym_STR_BANG] = SHIFT(217),
},
[218] = {
[sym__line_break] = REDUCE(sym_bool_op, 2),
[sym__line_break] = REDUCE_FRAGILE(sym_bool_op, 2),
[sym_comment] = SHIFT_EXTRA(),
[aux_sym_STR_LPAREN] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_SEMI] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_STAR] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_DOT] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_SLASH] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_PLUS] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_DASH] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_PIPE_PIPE] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_AMP_AMP] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_EQ_EQ] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_LT_EQ] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_LT] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_GT_EQ] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_GT] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_LPAREN] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_SEMI] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_STAR] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_DOT] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_SLASH] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_PLUS] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_DASH] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_PIPE_PIPE] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_AMP_AMP] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_EQ_EQ] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_LT_EQ] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_LT] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_GT_EQ] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_GT] = REDUCE_FRAGILE(sym_bool_op, 2),
},
[219] = {
[sym__line_break] = REDUCE(sym_var_name, 1),
@ -4777,23 +4777,23 @@ static const TSParseAction ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[aux_sym_STR_BANG] = SHIFT(223),
},
[224] = {
[sym_block_statement] = REDUCE(sym_bool_op, 2),
[sym_block_statement] = REDUCE_FRAGILE(sym_bool_op, 2),
[sym__line_break] = SHIFT_EXTRA(),
[sym_comment] = SHIFT_EXTRA(),
[aux_sym_STR_LPAREN] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_LBRACE] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_STAR] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_DOT] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_SLASH] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_PLUS] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_DASH] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_PIPE_PIPE] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_AMP_AMP] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_EQ_EQ] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_LT_EQ] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_LT] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_GT_EQ] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_GT] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_LPAREN] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_LBRACE] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_STAR] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_DOT] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_SLASH] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_PLUS] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_DASH] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_PIPE_PIPE] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_AMP_AMP] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_EQ_EQ] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_LT_EQ] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_LT] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_GT_EQ] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_GT] = REDUCE_FRAGILE(sym_bool_op, 2),
},
[225] = {
[sym_var_declaration] = REDUCE(sym_if_statement, 3),
@ -5217,22 +5217,22 @@ static const TSParseAction ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[248] = {
[sym__line_break] = SHIFT_EXTRA(),
[sym_comment] = SHIFT_EXTRA(),
[aux_sym_call_expression_repeat0] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_LPAREN] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_RPAREN] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_STAR] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_COMMA] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_DOT] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_SLASH] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_PLUS] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_DASH] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_PIPE_PIPE] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_AMP_AMP] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_EQ_EQ] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_LT_EQ] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_LT] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_GT_EQ] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_GT] = REDUCE(sym_bool_op, 2),
[aux_sym_call_expression_repeat0] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_LPAREN] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_RPAREN] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_STAR] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_COMMA] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_DOT] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_SLASH] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_PLUS] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_DASH] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_PIPE_PIPE] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_AMP_AMP] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_EQ_EQ] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_LT_EQ] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_LT] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_GT_EQ] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_GT] = REDUCE_FRAGILE(sym_bool_op, 2),
},
[249] = {
[sym__line_break] = SHIFT_EXTRA(),
@ -6340,24 +6340,24 @@ static const TSParseAction ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[aux_sym_STR_BANG] = SHIFT(316),
},
[317] = {
[sym__line_break] = REDUCE(sym_bool_op, 2),
[sym__line_break] = REDUCE_FRAGILE(sym_bool_op, 2),
[sym_comment] = SHIFT_EXTRA(),
[aux_sym_return_statement_repeat0] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_LPAREN] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_SEMI] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_STAR] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_COMMA] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_DOT] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_SLASH] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_PLUS] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_DASH] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_PIPE_PIPE] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_AMP_AMP] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_EQ_EQ] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_LT_EQ] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_LT] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_GT_EQ] = REDUCE(sym_bool_op, 2),
[aux_sym_STR_GT] = REDUCE(sym_bool_op, 2),
[aux_sym_return_statement_repeat0] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_LPAREN] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_SEMI] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_STAR] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_COMMA] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_DOT] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_SLASH] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_PLUS] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_DASH] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_PIPE_PIPE] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_AMP_AMP] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_EQ_EQ] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_LT_EQ] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_LT] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_GT_EQ] = REDUCE_FRAGILE(sym_bool_op, 2),
[aux_sym_STR_GT] = REDUCE_FRAGILE(sym_bool_op, 2),
},
[318] = {
[sym_var_declaration] = REDUCE(sym_return_statement, 3),

2382
spec/fixtures/parsers/javascript.c vendored
View file

File diff suppressed because it is too large Load diff

71
src/compiler/build_tables/action_takes_precedence.cc
View file

@ -1,70 +1,63 @@
#include "compiler/build_tables/action_takes_precedence.h"
#include <iostream>
namespace tree_sitter {
namespace build_tables {
using std::pair;
pair<bool, bool> action_takes_precedence(const ParseAction &new_action,
const ParseAction &old_action,
const rules::Symbol &symbol) {
pair<bool, ConflictType>
action_takes_precedence(const ParseAction &new_action,
const ParseAction &old_action,
const rules::Symbol &symbol) {
if (new_action.type < old_action.type) {
auto opposite =
action_takes_precedence(old_action, new_action, symbol);
return { !opposite.first, opposite.second };
}
bool has_precedence = false, has_conflict = false;
switch (old_action.type) {
case ParseActionTypeError:
has_precedence = true;
break;
return { true, ConflictTypeNone };
case ParseActionTypeShift: {
int min_precedence = *old_action.precedence_values.begin();
int max_precedence = *old_action.precedence_values.rbegin();
switch (new_action.type) {
case ParseActionTypeReduce: {
int new_precedence = *new_action.precedence_values.rbegin();
if (new_precedence < max_precedence) {
if (new_precedence > min_precedence)
has_conflict = true;
} else if (new_precedence > max_precedence) {
has_precedence = true;
} else {
has_conflict = true;
}
break;
case ParseActionTypeShift:
if (new_action.type == ParseActionTypeReduce) {
int min_precedence = *old_action.precedence_values.begin();
int max_precedence = *old_action.precedence_values.rbegin();
int new_precedence = *new_action.precedence_values.rbegin();
if (new_precedence < min_precedence)
return { false, ConflictTypeResolved };
else if (new_precedence > max_precedence)
return { true, ConflictTypeResolved };
else {
// TODO: Add associativity annotations. In the event of a precedence
// tie, return ConflictTypeError unless there is an associativity
// annotation to break the tie.
return { false, ConflictTypeResolved };
}
default:
break;
}
break;
}
case ParseActionTypeReduce:
switch (new_action.type) {
case ParseActionTypeReduce: {
int old_precedence = *old_action.precedence_values.begin();
int new_precedence = *new_action.precedence_values.begin();
if (new_precedence > old_precedence) {
has_precedence = true;
} else if (new_precedence == old_precedence) {
has_precedence = new_action.symbol.index < old_action.symbol.index;
has_conflict = true;
}
break;
if (new_action.type == ParseActionTypeReduce) {
int old_precedence = *old_action.precedence_values.begin();
int new_precedence = *new_action.precedence_values.begin();
if (new_precedence > old_precedence) {
return { true, ConflictTypeResolved };
} else if (new_precedence < old_precedence) {
return { false, ConflictTypeResolved };
} else {
return { false, ConflictTypeError };
}
default:
break;
}
default:
break;
}
return { has_precedence, has_conflict };
return { false, ConflictTypeNone };
}
bool action_takes_precedence(const LexAction &new_action,

13
src/compiler/build_tables/action_takes_precedence.h
View file

@ -10,9 +10,16 @@
namespace tree_sitter {
namespace build_tables {
std::pair<bool, bool> action_takes_precedence(const ParseAction &new_action,
const ParseAction &old_action,
const rules::Symbol &symbol);
enum ConflictType {
ConflictTypeNone,
ConflictTypeResolved,
ConflictTypeError
};
std::pair<bool, ConflictType>
action_takes_precedence(const ParseAction &new_action,
const ParseAction &old_action,
const rules::Symbol &symbol);
bool action_takes_precedence(const LexAction &new_action,
const LexAction &old_action);

11
src/compiler/build_tables/build_conflict.cc
View file

@ -57,17 +57,18 @@ static string action_description(const ParseAction &action,
}
}
Conflict build_conflict(const ParseAction &left, const ParseAction &right,
string build_conflict(const ParseAction &left, const ParseAction &right,
const ParseItemSet &item_set, const Symbol &sym,
const SyntaxGrammar &grammar,
const LexicalGrammar &lex_grammar) {
if (right < left)
return build_conflict(right, left, item_set, sym, grammar, lex_grammar);
return Conflict(symbol_name(sym, grammar, lex_grammar) + ": " +
action_description(left, item_set, grammar, lex_grammar) +
" / " +
action_description(right, item_set, grammar, lex_grammar));
return symbol_name(sym, grammar, lex_grammar) +
": " +
action_description(left, item_set, grammar, lex_grammar) +
" / " +
action_description(right, item_set, grammar, lex_grammar);
}
} // namespace build_tables

6
src/compiler/build_tables/build_conflict.h
View file

@ -13,9 +13,9 @@ class LexicalGrammar;
namespace build_tables {
Conflict build_conflict(const ParseAction &left, const ParseAction &right,
const ParseItemSet &item_set, const rules::Symbol &,
const SyntaxGrammar &, const LexicalGrammar &);
std::string build_conflict(const ParseAction &, const ParseAction &,
const ParseItemSet &, const rules::Symbol &,
const SyntaxGrammar &, const LexicalGrammar &);
} // namespace build_tables
} // namespace tree_sitter

40
src/compiler/build_tables/build_parse_table.cc
View file

@ -35,14 +35,14 @@ class ParseTableBuilder {
vector<vector<Symbol>> productions;
vector<pair<ParseItemSet, ParseStateId>> item_sets_to_process;
ParseTable parse_table;
std::set<Conflict> conflicts;
std::set<string> conflicts;
public:
ParseTableBuilder(const SyntaxGrammar &grammar,
const LexicalGrammar &lex_grammar)
: grammar(grammar), lex_grammar(lex_grammar) {}
pair<ParseTable, vector<Conflict>> build() {
pair<ParseTable, const GrammarError *> build() {
auto start_symbol = grammar.rules.empty()
? make_shared<Symbol>(0, rules::SymbolOptionToken)
: make_shared<Symbol>(0);
@ -59,6 +59,12 @@ class ParseTableBuilder {
add_reduce_actions(item_set, state_id);
add_shift_actions(item_set, state_id);
add_shift_extra_actions(state_id);
if (!conflicts.empty())
return {
parse_table,
new GrammarError(GrammarErrorTypeParseConflict, *conflicts.begin())
};
}
for (ParseStateId state = 0; state < parse_table.states.size(); state++)
@ -67,7 +73,7 @@ class ParseTableBuilder {
parse_table.symbols.insert(rules::ERROR());
parse_table.symbols.insert(rules::DOCUMENT());
return { parse_table, conflicts_vector() };
return { parse_table, nullptr };
}
private:
@ -162,13 +168,18 @@ class ParseTableBuilder {
auto result = action_takes_precedence(action, current_action->second,
symbol);
if (result.second) {
record_conflict(symbol, current_action->second, action, item_set);
if (action.type == ParseActionTypeReduce)
parse_table.fragile_production_ids.insert(action.production_id);
if (current_action->second.type == ParseActionTypeReduce)
parse_table.fragile_production_ids.insert(current_action->second.production_id);
switch (result.second) {
case ConflictTypeResolved:
if (action.type == ParseActionTypeReduce)
parse_table.fragile_production_ids.insert(action.production_id);
if (current_action->second.type == ParseActionTypeReduce)
parse_table.fragile_production_ids.insert(current_action->second.production_id);
break;
case ConflictTypeError:
record_conflict(symbol, current_action->second, action, item_set);
break;
default:
break;
}
return result.first;
@ -197,18 +208,17 @@ class ParseTableBuilder {
void record_conflict(const Symbol &sym, const ParseAction &left,
const ParseAction &right, const ParseItemSet &item_set) {
conflicts.insert(
build_conflict(left, right, item_set, sym, grammar, lex_grammar));
conflicts.insert(build_conflict(left, right, item_set, sym, grammar, lex_grammar));
}
vector<Conflict> conflicts_vector() const {
vector<Conflict> result;
vector<string> conflicts_vector() const {
vector<string> result;
result.insert(result.end(), conflicts.begin(), conflicts.end());
return result;
}
};
pair<ParseTable, vector<Conflict>> build_parse_table(
pair<ParseTable, const GrammarError *> build_parse_table(
const SyntaxGrammar &grammar, const LexicalGrammar &lex_grammar) {
return ParseTableBuilder(grammar, lex_grammar).build();
}

4
src/compiler/build_tables/build_parse_table.h
View file

@ -13,8 +13,8 @@ class LexicalGrammar;
namespace build_tables {
std::pair<ParseTable, std::vector<Conflict>> build_parse_table(
const SyntaxGrammar &, const LexicalGrammar &);
std::pair<ParseTable, const GrammarError *>
build_parse_table(const SyntaxGrammar &, const LexicalGrammar &);
} // namespace build_tables
} // namespace tree_sitter

9
src/compiler/build_tables/build_tables.cc
View file

@ -6,17 +6,18 @@
namespace tree_sitter {
namespace build_tables {
using std::string;
using std::tuple;
using std::vector;
using std::make_tuple;
tuple<ParseTable, LexTable, vector<Conflict>> build_tables(
const SyntaxGrammar &grammar, const LexicalGrammar &lex_grammar) {
tuple<ParseTable, LexTable, const GrammarError *>
build_tables(const SyntaxGrammar &grammar, const LexicalGrammar &lex_grammar) {
auto parse_table_result = build_parse_table(grammar, lex_grammar);
ParseTable parse_table = parse_table_result.first;
vector<Conflict> conflicts = parse_table_result.second;
const GrammarError *error = parse_table_result.second;
LexTable lex_table = build_lex_table(&parse_table, lex_grammar);
return make_tuple(parse_table, lex_table, conflicts);
return make_tuple(parse_table, lex_table, error);
}
} // namespace build_tables

5
src/compiler/build_tables/build_tables.h
View file

@ -1,6 +1,7 @@
#ifndef COMPILER_BUILD_TABLES_BUILD_TABLES_H_
#define COMPILER_BUILD_TABLES_BUILD_TABLES_H_
#include <string>
#include <utility>
#include <vector>
#include "tree_sitter/compiler.h"
@ -14,8 +15,8 @@ class LexicalGrammar;
namespace build_tables {
std::tuple<ParseTable, LexTable, std::vector<Conflict>> build_tables(
const SyntaxGrammar &, const LexicalGrammar &);
std::tuple<ParseTable, LexTable, const GrammarError *>
build_tables(const SyntaxGrammar &, const LexicalGrammar &);
} // namespace build_tables
} // namespace tree_sitter

15
src/compiler/compile.cc
View file

@ -7,32 +7,35 @@
namespace tree_sitter {
using std::tuple;
using std::pair;
using std::string;
using std::vector;
using std::get;
using std::make_tuple;
tuple<string, vector<Conflict>, const GrammarError *> compile(
const Grammar &grammar, std::string name) {
pair<string, const GrammarError *>
compile(const Grammar &grammar, std::string name) {
auto prepare_grammar_result = prepare_grammar::prepare_grammar(grammar);
const SyntaxGrammar &syntax_grammar = get<0>(prepare_grammar_result);
const LexicalGrammar &lexical_grammar = get<1>(prepare_grammar_result);
const GrammarError *error = get<2>(prepare_grammar_result);
if (error)
return make_tuple("", vector<Conflict>(), error);
return {"", error};
auto table_build_result =
build_tables::build_tables(syntax_grammar, lexical_grammar);
const ParseTable &parse_table = get<0>(table_build_result);
const LexTable &lex_table = get<1>(table_build_result);
const vector<Conflict> &conflicts = get<2>(table_build_result);
error = get<2>(table_build_result);
if (error)
return {"", error};
string code = generate_code::c_code(name, parse_table, lex_table,
syntax_grammar, lexical_grammar);
return make_tuple(code, conflicts, nullptr);
return {code, nullptr};
}
} // namespace tree_sitter

22
src/compiler/conflict.cc
View file

@ -1,22 +0,0 @@
#include <string>
#include "tree_sitter/compiler.h"
namespace tree_sitter {
using std::string;
Conflict::Conflict(string description) : description(description) {}
bool Conflict::operator==(const tree_sitter::Conflict &other) const {
return other.description == description;
}
bool Conflict::operator<(const tree_sitter::Conflict &other) const {
return other.description < description;
}
std::ostream &operator<<(std::ostream &stream, const Conflict &conflict) {
return stream << "#<conflict " + conflict.description + ">";
}
} // namespace tree_sitter

4
src/compiler/rules/rules.cc
View file

@ -58,11 +58,11 @@ rule_ptr err(const rule_ptr &rule) {
return choice({ rule, ERROR().copy() });
}
rule_ptr prec(int precedence, rule_ptr rule) {
rule_ptr prec(int precedence, const rule_ptr &rule) {
return metadata(rule, { { PRECEDENCE, precedence } });
}
rule_ptr token(rule_ptr rule) {
rule_ptr token(const rule_ptr &rule) {
return metadata(rule, { { IS_TOKEN, 1 } });
}