traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Respect expected_conflicts field when building parse table

Browse source
This commit is contained in:
Max Brunsfeld 2015-06-28 16:22:31 -05:00
parent c9a482bbf3
commit aabcb10cfb
15 changed files with 5054 additions and 348 deletions
Download patch file Download diff file Expand all files Collapse all files

26
spec/compiler/build_tables/build_parse_table_spec.cc
View file

@ -25,18 +25,18 @@ describe("build_parse_table", []() {
it("first looks for the start rule and its item set closure", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
AssertThat(result.first.states[0].actions, Equals(map<Symbol, ParseAction>({
AssertThat(result.first.states[0].actions, Equals(map<Symbol, vector<ParseAction>>({
// start item
{ Symbol(0), ParseAction::Shift(1, { 0 }) },
{ Symbol(0), {ParseAction::Shift(1, { 0 })} },
// expanded from the item set closure of the start item
{ Symbol(1), ParseAction::Shift(2, { 0 }) },
{ Symbol(2), ParseAction::Shift(2, { 0 }) },
{ Symbol(0, SymbolOptionToken), ParseAction::Shift(3, { 0 }) },
{ Symbol(1, SymbolOptionToken), ParseAction::Shift(4, { 0 }) },
{ Symbol(1), {ParseAction::Shift(2, { 0 })} },
{ Symbol(2), {ParseAction::Shift(2, { 0 })} },
{ Symbol(0, SymbolOptionToken), {ParseAction::Shift(3, { 0 })} },
{ Symbol(1, SymbolOptionToken), {ParseAction::Shift(4, { 0 })} },
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
{ Symbol(2, SymbolOptionToken), {ParseAction::ShiftExtra()} },
})));
});
@ -48,22 +48,22 @@ describe("build_parse_table", []() {
it("accepts the input when EOF occurs after the start rule", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
AssertThat(result.first.states[1].actions, Equals(map<Symbol, ParseAction>({
{ END_OF_INPUT(), ParseAction::Accept() },
AssertThat(result.first.states[1].actions, Equals(map<Symbol, vector<ParseAction>>({
{ END_OF_INPUT(), {ParseAction::Accept()} },
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
{ Symbol(2, SymbolOptionToken), {ParseAction::ShiftExtra()} },
})));
});
it("reduces a rule once it has been consumed", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
AssertThat(result.first.states[2].actions, Equals(map<Symbol, ParseAction>({
{ END_OF_INPUT(), ParseAction::Reduce(Symbol(0), 1, 0, AssociativityLeft, 0) },
AssertThat(result.first.states[2].actions, Equals(map<Symbol, vector<ParseAction>>({
{ END_OF_INPUT(), {ParseAction::Reduce(Symbol(0), 1, 0, AssociativityLeft, 0)} },
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
{ Symbol(2, SymbolOptionToken), {ParseAction::ShiftExtra()} },
})));
});
});

205
spec/compiler/build_tables/parse_conflict_manager_spec.cc
View file

@ -18,59 +18,48 @@ describe("ParseConflictManager", []() {
{ "other_rule2", i_token(0) },
}, {}, { Symbol(2, SymbolOptionToken) }, set<set<Symbol>>());
LexicalGrammar lexical_grammar({
{ "other_token", pattern("[a-b]") },
{ "lookahead_token", pattern("[c-d]") },
}, {});
tuple<bool, ConflictType, string> result;
pair<bool, ConflictType> result;
Symbol sym1(0);
Symbol sym2(1);
Symbol lookahead_sym(1, SymbolOptionToken);
ParseConflictManager *conflict_manager;
before_each([&]() {
conflict_manager = new ParseConflictManager(syntax_grammar, lexical_grammar);
conflict_manager = new ParseConflictManager(syntax_grammar);
});
after_each([&]() {
delete conflict_manager;
});
describe(".get_production_id", [&]() {
it("returns different IDs for different productions", [&]() {
int id1 = conflict_manager->get_production_id(vector<Symbol>({ Symbol(1), Symbol(2) }));
AssertThat(id1, Equals(0));
int id2 = conflict_manager->get_production_id(vector<Symbol>({ Symbol(1), Symbol(2), Symbol(3) }));
AssertThat(id2, Equals(1));
int id3 = conflict_manager->get_production_id(vector<Symbol>({ Symbol(1) }));
AssertThat(id3, Equals(2));
int id4 = conflict_manager->get_production_id(vector<Symbol>({ Symbol(1), Symbol(2) }));
AssertThat(id4, Equals(id1));
});
});
describe(".resolve", [&]() {
ParseItemSet item_set({
{ ParseItem(Symbol(0), blank(), { Symbol(0, SymbolOptionToken) }), set<Symbol>() }, // in_progress_rule1
{ ParseItem(Symbol(3), blank(), {}), set<Symbol>() }, // other_rule1
});
describe("errors", [&]() {
ParseAction error = ParseAction::Error();
ParseAction non_error = ParseAction::Shift(2, { 0 });
it("favors non-errors and reports no conflict", [&]() {
result = conflict_manager->resolve(non_error, error, sym1, item_set);
AssertThat(get<0>(result), IsTrue());
AssertThat(get<1>(result), Equals(ConflictTypeNone));
result = conflict_manager->resolve(non_error, error, sym1);
AssertThat(result.first, IsTrue());
AssertThat(result.second, Equals(ConflictTypeNone));
result = conflict_manager->resolve(error, non_error, sym1, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeNone));
result = conflict_manager->resolve(error, non_error, sym1);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeNone));
});
});
describe("shift-extra actions", [&]() {
ParseAction shift_extra = ParseAction::Error();
ParseAction other = ParseAction::Shift(2, { 0 });
it("favors other actions over shift-extra actions", [&]() {
result = conflict_manager->resolve(other, shift_extra, sym1);
AssertThat(result.first, IsTrue());
AssertThat(result.second, Equals(ConflictTypeNone));
result = conflict_manager->resolve(shift_extra, other, sym1);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeNone));
});
});
@ -80,13 +69,13 @@ describe("ParseConflictManager", []() {
ParseAction reduce = ParseAction::Reduce(sym2, 1, 1, AssociativityLeft, 0);
it("favors the shift and reports the conflict as resolved", [&]() {
result = conflict_manager->resolve(shift, reduce, sym1, item_set);
AssertThat(get<0>(result), IsTrue());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(shift, reduce, sym1);
AssertThat(result.first, IsTrue());
AssertThat(result.second, Equals(ConflictTypeResolved));
result = conflict_manager->resolve(reduce, shift, sym1, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(reduce, shift, sym1);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeResolved));
});
});
@ -97,13 +86,13 @@ describe("ParseConflictManager", []() {
ParseAction reduce = ParseAction::Reduce(sym2, 1, 3, AssociativityLeft, 0);
it("favors the reduce and reports the conflict as resolved", [&]() {
result = conflict_manager->resolve(shift, reduce, sym1, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(shift, reduce, sym1);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeResolved));
result = conflict_manager->resolve(reduce, shift, sym1, item_set);
AssertThat(get<0>(result), IsTrue());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(reduce, shift, sym1);
AssertThat(result.first, IsTrue());
AssertThat(result.second, Equals(ConflictTypeResolved));
});
});
@ -112,13 +101,13 @@ describe("ParseConflictManager", []() {
ParseAction reduce = ParseAction::Reduce(sym2, 1, 0, AssociativityLeft, 0);
it("favors the reduce and reports the conflict as resolved", [&]() {
result = conflict_manager->resolve(reduce, shift, sym1, item_set);
AssertThat(get<0>(result), IsTrue());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(reduce, shift, sym1);
AssertThat(result.first, IsTrue());
AssertThat(result.second, Equals(ConflictTypeResolved));
result = conflict_manager->resolve(shift, reduce, sym1, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(shift, reduce, sym1);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeResolved));
});
});
@ -127,13 +116,13 @@ describe("ParseConflictManager", []() {
ParseAction reduce = ParseAction::Reduce(sym2, 1, 0, AssociativityRight, 0);
it("favors the shift, and reports the conflict as resolved", [&]() {
result = conflict_manager->resolve(reduce, shift, sym1, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(reduce, shift, sym1);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeResolved));
result = conflict_manager->resolve(shift, reduce, sym1, item_set);
AssertThat(get<0>(result), IsTrue());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(shift, reduce, sym1);
AssertThat(result.first, IsTrue());
AssertThat(result.second, Equals(ConflictTypeResolved));
});
});
@ -142,31 +131,12 @@ describe("ParseConflictManager", []() {
ParseAction shift = ParseAction::Shift(2, { 0 });
ParseAction reduce = ParseAction::Reduce(Symbol(2), 1, 0, AssociativityUnspecified, 0);
reduce.production_id = conflict_manager->get_production_id(vector<Symbol>({
Symbol(3),
Symbol(4),
}));
result = conflict_manager->resolve(reduce, shift, lookahead_sym);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeUnresolved));
result = conflict_manager->resolve(reduce, shift, lookahead_sym, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeError));
AssertThat(get<2>(result), Equals(
"Within: in_progress_rule1\n"
"Lookahead: lookahead_token\n"
"Possible Actions:\n"
"* Shift (Precedence 0)\n"
"* Reduce other_rule1 other_rule2 -> reduced_rule (Precedence 0)"
));
result = conflict_manager->resolve(shift, reduce, lookahead_sym, item_set);
AssertThat(get<0>(result), IsTrue());
AssertThat(get<2>(result), Equals(
"Within: in_progress_rule1\n"
"Lookahead: lookahead_token\n"
"Possible Actions:\n"
"* Shift (Precedence 0)\n"
"* Reduce other_rule1 other_rule2 -> reduced_rule (Precedence 0)"
));
result = conflict_manager->resolve(shift, reduce, lookahead_sym);
AssertThat(result.first, IsTrue());
});
});
@ -175,25 +145,13 @@ describe("ParseConflictManager", []() {
ParseAction reduce = ParseAction::Reduce(Symbol(2), 1, 2, AssociativityLeft, 0);
it("returns false and reports an unresolved conflict", [&]() {
reduce.production_id = conflict_manager->get_production_id(vector<Symbol>({
Symbol(3),
Symbol(4),
}));
result = conflict_manager->resolve(reduce, shift, lookahead_sym);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeUnresolved));
result = conflict_manager->resolve(reduce, shift, lookahead_sym, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeError));
result = conflict_manager->resolve(shift, reduce, lookahead_sym, item_set);
AssertThat(get<0>(result), IsTrue());
AssertThat(get<1>(result), Equals(ConflictTypeError));
AssertThat(get<2>(result), Equals(
"Within: in_progress_rule1\n"
"Lookahead: lookahead_token\n"
"Possible Actions:\n"
"* Shift (Precedences 0, 3)\n"
"* Reduce other_rule1 other_rule2 -> reduced_rule (Precedence 2)"
));
result = conflict_manager->resolve(shift, reduce, lookahead_sym);
AssertThat(result.first, IsTrue());
AssertThat(result.second, Equals(ConflictTypeUnresolved));
});
});
});
@ -204,13 +162,13 @@ describe("ParseConflictManager", []() {
ParseAction right = ParseAction::Reduce(sym2, 1, 3, AssociativityLeft, 0);
it("favors that action", [&]() {
result = conflict_manager->resolve(left, right, sym1, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(left, right, sym1);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeResolved));
result = conflict_manager->resolve(right, left, sym1, item_set);
AssertThat(get<0>(result), IsTrue());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(right, left, sym1);
AssertThat(result.first, IsTrue());
AssertThat(result.second, Equals(ConflictTypeResolved));
});
});
@ -219,36 +177,13 @@ describe("ParseConflictManager", []() {
ParseAction left = ParseAction::Reduce(Symbol(2), 1, 0, AssociativityLeft, 0);
ParseAction right = ParseAction::Reduce(Symbol(3), 1, 0, AssociativityLeft, 0);
left.production_id = conflict_manager->get_production_id(vector<Symbol>({
Symbol(3),
Symbol(4),
}));
result = conflict_manager->resolve(right, left, lookahead_sym);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeUnresolved));
right.production_id = conflict_manager->get_production_id(vector<Symbol>({
Symbol(4),
}));
result = conflict_manager->resolve(right, left, lookahead_sym, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeError));
AssertThat(get<2>(result), Equals(
"Within: in_progress_rule1\n"
"Lookahead: lookahead_token\n"
"Possible Actions:\n"
"* Reduce other_rule1 other_rule2 -> reduced_rule (Precedence 0)\n"
"* Reduce other_rule2 -> other_rule1 (Precedence 0)"
));
result = conflict_manager->resolve(left, right, lookahead_sym, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeError));
AssertThat(get<2>(result), Equals(
"Within: in_progress_rule1\n"
"Lookahead: lookahead_token\n"
"Possible Actions:\n"
"* Reduce other_rule2 -> other_rule1 (Precedence 0)\n"
"* Reduce other_rule1 other_rule2 -> reduced_rule (Precedence 0)"
));
result = conflict_manager->resolve(left, right, lookahead_sym);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeUnresolved));
});
});
});

2
spec/compiler/compile_examples.cc
View file

@ -14,6 +14,7 @@ extern const Grammar arithmetic;
extern const Grammar javascript;
extern const Grammar json;
extern const Grammar golang;
extern const Grammar c;
} // namespace tree_sitter_examples
@ -40,6 +41,7 @@ describe("compiling the example grammars", []() {
compile_grammar(tree_sitter_examples::json, "json");
compile_grammar(tree_sitter_examples::javascript, "javascript");
compile_grammar(tree_sitter_examples::golang, "golang");
compile_grammar(tree_sitter_examples::c, "c");
});
END_TEST

164
spec/fixtures/grammars/c.cc vendored Normal file
View file

@ -0,0 +1,164 @@
#include "tree_sitter/compiler.h"
#include "helpers.h"
namespace tree_sitter_examples {
using tree_sitter::Grammar;
using namespace tree_sitter::rules;
// http://slps.github.io/zoo/c/iso-9899-tc3.html
extern const Grammar c = Grammar({
{ "program", choice({
sym("function_definition"),
sym("declaration") }) },
{ "function_definition", seq({
optional(sym("declaration_specifiers")),
sym("type_specifier"),
sym("declarator"),
repeat(sym("declaration")),
sym("compound_statement") }) },
{ "declaration_specifiers", repeat1(choice({
sym("storage_class_specifier"),
sym("type_qualifier") })) },
{ "storage_class_specifier", choice({
str("typedef"),
str("extern"),
str("static"),
str("auto"),
str("register") }) },
{ "type_specifier", choice({
sym("struct_specifier"),
seq({
repeat(choice({
str("signed"),
str("unsigned"),
str("long"),
str("short") })),
sym("identifier") }) }) },
{ "struct_specifier", seq({
str("struct"),
optional(sym("identifier")),
seq({
str("{"),
repeat(sym("struct_declaration")),
str("}") }) }) },
{ "struct_declaration", seq({
sym("type_specifier"),
sym("declarator") }) },
{ "parameter_declaration", seq({
optional(sym("declaration_specifiers")),
sym("type_specifier"),
sym("declarator") }) },
{ "declaration", seq({
optional(sym("declaration_specifiers")),
sym("type_specifier"),
comma_sep1(sym("init_declarator")),
str(";") }) },
{ "init_declarator", choice({
sym("declarator"),
seq({ sym("declarator"), str("="), sym("initializer") }) }) },
{ "initializer", choice({
sym("expression"),
seq({
str("{"),
sym("initializer_list"),
optional(str(",")),
str("}") }) }) },
{ "initializer_list", choice({
seq({
optional(sym("designation")),
sym("initializer") }),
seq({
sym("initializer_list"),
str(","),
optional(sym("designation")),
sym("initializer") }) }) },
{ "designation", seq({
repeat1(choice({
seq({
str("["),
sym("expression"),
str("]") }),
seq({
str("."),
sym("identifier") }) })),
str("=") }) },
{ "declarator", seq({
repeat(sym("star")),
sym("direct_declarator") }) },
{ "direct_declarator", choice({
sym("identifier"),
seq({
str("("),
sym("declarator"),
str(")") }),
seq({
sym("direct_declarator"),
str("["),
optional(sym("expression")),
str("]") }),
seq({
sym("direct_declarator"),
str("("),
comma_sep(sym("parameter_declaration")),
str(")") }) }) },
{ "type_qualifier", choice({
str("const"),
str("restrict"),
str("volatile") }) },
{ "star", str("*") },
{ "compound_statement", seq({
str("{"),
repeat(choice({ sym("declaration"), sym("statement") })),
str("}") }) },
{ "expression", choice({
sym("math_expression"),
sym("string"),
sym("identifier"),
sym("number") }) },
{ "math_expression", choice({
prec(1, seq({ sym("expression"), str("+"), sym("expression") })),
prec(2, seq({ sym("expression"), sym("star"), sym("expression") })) }) },
{ "statement", choice({
sym("expression_statement") }) },
{ "expression_statement", seq({
sym("expression"),
str(";") }) },
{ "string", delimited("\"") },
{ "identifier", pattern("\\a[\\w_]*") },
{ "number", pattern("\\d+(\\.\\d+)?") },
{ "comment", pattern("//[^\n]*") },
}).ubiquitous_tokens({
sym("comment"),
pattern("[ \t\r\n]"),
}).expected_conflicts({
{ "type_specifier", "expression" },
});
} // namespace tree_sitter_examples

4
spec/fixtures/grammars/helpers.cc vendored
View file

@ -4,6 +4,10 @@ namespace tree_sitter_examples {
using namespace tree_sitter::rules;
rule_ptr repeat1(rule_ptr element) {
return seq({ element, repeat(element) });
}
rule_ptr comma_sep1(rule_ptr element) {
return seq({ element, repeat(seq({ str(","), element })) });
}

1
spec/fixtures/grammars/helpers.h vendored
View file

@ -7,6 +7,7 @@ namespace tree_sitter_examples {
using namespace tree_sitter::rules;
rule_ptr repeat1(rule_ptr element);
rule_ptr comma_sep1(rule_ptr element);
rule_ptr comma_sep(rule_ptr element);
rule_ptr optional(rule_ptr rule);

4535
spec/fixtures/parsers/c.c vendored Normal file
View file

File diff suppressed because it is too large Load diff

38
spec/runtime/languages/c/declarations.txt Normal file
View file

@ -0,0 +1,38 @@
==========================================
simple declarations
==========================================
int x;
---
(declaration (identifier) (identifier))
==========================================
simple functions
==========================================
int main() {
}
---
(function_definition
(identifier) (direct_declarator (identifier))
(compound_statement))
==========================================
simple declarations in functions
==========================================
int main() {
x + y;
}
---
(function_definition
(identifier)
(direct_declarator (identifier))
(compound_statement
(expression_statement (math_expression (identifier) (identifier)))))

4
spec/runtime/languages/language_specs.cc
View file

@ -8,6 +8,7 @@ extern "C" const TSLanguage *ts_language_javascript();
extern "C" const TSLanguage *ts_language_json();
extern "C" const TSLanguage *ts_language_arithmetic();
extern "C" const TSLanguage *ts_language_golang();
extern "C" const TSLanguage *ts_language_c();
START_TEST
@ -23,7 +24,7 @@ describe("Languages", [&]() {
});
auto run_tests_for_language = [&](string language_name, const TSLanguage *language) {
describe(language_name.c_str(), [&]() {
describe((string("The ") + language_name + " parser").c_str(), [&]() {
before_each([&]() {
ts_document_set_language(doc, language);
// ts_document_set_debugger(doc, log_debugger_make());
@ -85,6 +86,7 @@ describe("Languages", [&]() {
run_tests_for_language("arithmetic", ts_language_arithmetic());
run_tests_for_language("javascript", ts_language_javascript());
run_tests_for_language("golang", ts_language_golang());
run_tests_for_language("c", ts_language_c());
});
END_TEST

197
src/compiler/build_tables/build_parse_table.cc
View file

@ -16,20 +16,22 @@
namespace tree_sitter {
namespace build_tables {
using std::get;
using std::pair;
using std::vector;
using std::set;
using std::map;
using std::string;
using std::to_string;
using std::unordered_map;
using std::make_shared;
using rules::Symbol;
class ParseTableBuilder {
const SyntaxGrammar grammar;
const LexicalGrammar lexical_grammar;
ParseConflictManager conflict_manager;
unordered_map<const ParseItemSet, ParseStateId> parse_state_ids;
vector<vector<Symbol>> productions;
vector<pair<ParseItemSet, ParseStateId>> item_sets_to_process;
ParseTable parse_table;
std::set<string> conflicts;
@ -37,7 +39,7 @@ class ParseTableBuilder {
public:
ParseTableBuilder(const SyntaxGrammar &grammar,
const LexicalGrammar &lex_grammar)
: grammar(grammar), conflict_manager(grammar, lex_grammar) {}
: grammar(grammar), lexical_grammar(lex_grammar), conflict_manager(grammar) {}
pair<ParseTable, const GrammarError *> build() {
auto start_symbol = grammar.rules.empty()
@ -55,7 +57,7 @@ class ParseTableBuilder {
add_reduce_actions(item_set, state_id);
add_shift_actions(item_set, state_id);
add_shift_extra_actions(state_id);
add_shift_extra_actions(item_set, state_id);
if (!conflicts.empty())
return {
@ -92,13 +94,11 @@ class ParseTableBuilder {
const Symbol &symbol = transition.first;
const ParseItemSet &next_item_set = transition.second;
ParseAction new_action =
ParseAction::Shift(0, precedence_values_for_item_set(next_item_set));
if (should_add_action(state_id, item_set, symbol, new_action)) {
ParseStateId new_state_id = add_parse_state(next_item_set);
new_action.state_index = new_state_id;
parse_table.add_action(state_id, symbol, new_action);
}
ParseAction *new_action = add_action(state_id, symbol,
ParseAction::Shift(0, precedence_values_for_item_set(next_item_set)),
item_set);
if (new_action)
new_action->state_index = add_parse_state(next_item_set);
}
}
@ -113,73 +113,85 @@ class ParseTableBuilder {
? ParseAction::Accept()
: ParseAction::Reduce(item.lhs, item.consumed_symbols.size(),
item.precedence(), item.associativity(),
conflict_manager.get_production_id(item.consumed_symbols));
get_production_id(item.consumed_symbols));
for (const auto &lookahead_sym : lookahead_symbols)
if (should_add_action(state_id, item_set, lookahead_sym, action))
parse_table.add_action(state_id, lookahead_sym, action);
add_action(state_id, lookahead_sym, action, item_set);
}
}
}
void add_shift_extra_actions(ParseStateId state_id) {
const map<Symbol, ParseAction> &actions =
parse_table.states[state_id].actions;
for (const Symbol &ubiquitous_symbol : grammar.ubiquitous_tokens) {
const auto &pair_for_symbol = actions.find(ubiquitous_symbol);
if (pair_for_symbol == actions.end()) {
parse_table.add_action(state_id, ubiquitous_symbol,
ParseAction::ShiftExtra());
}
}
void add_shift_extra_actions(const ParseItemSet &item_set, ParseStateId state_id) {
for (const Symbol &ubiquitous_symbol : grammar.ubiquitous_tokens)
add_action(state_id, ubiquitous_symbol, ParseAction::ShiftExtra(), item_set);
}
void add_reduce_extra_actions(ParseStateId state_id) {
const map<Symbol, ParseAction> &actions =
const ParseItemSet item_set;
const map<Symbol, vector<ParseAction>> &actions =
parse_table.states[state_id].actions;
for (const Symbol &ubiquitous_symbol : grammar.ubiquitous_tokens) {
const auto &pair_for_symbol = actions.find(ubiquitous_symbol);
const auto &entry = actions.find(ubiquitous_symbol);
if (entry == actions.end())
continue;
if (pair_for_symbol != actions.end() &&
pair_for_symbol->second.type == ParseActionTypeShift) {
size_t shift_state_id = pair_for_symbol->second.state_index;
for (const auto &pair : actions) {
const Symbol &lookahead_sym = pair.first;
ParseAction reduce_extra = ParseAction::ReduceExtra(ubiquitous_symbol);
if (should_add_action(shift_state_id, ParseItemSet(), lookahead_sym, reduce_extra))
parse_table.add_action(shift_state_id, lookahead_sym, reduce_extra);
for (const auto &action : entry->second) {
if (action.type == ParseActionTypeShift) {
size_t shift_state_id = action.state_index;
for (const auto &pair : actions) {
const Symbol &lookahead_sym = pair.first;
ParseAction reduce_extra = ParseAction::ReduceExtra(ubiquitous_symbol);
add_action(shift_state_id, lookahead_sym, reduce_extra, item_set);
}
}
}
}
}
bool should_add_action(ParseStateId state_id, const ParseItemSet &item_set,
const Symbol &symbol, const ParseAction &action) {
ParseAction *add_action(ParseStateId state_id, Symbol lookahead_sym,
const ParseAction &action, const ParseItemSet &item_set) {
auto &current_actions = parse_table.states[state_id].actions;
auto current_action = current_actions.find(symbol);
if (current_action == current_actions.end())
return true;
auto current_entry = current_actions.find(lookahead_sym);
if (current_entry == current_actions.end())
return &parse_table.set_action(state_id, lookahead_sym, action);
auto result = conflict_manager.resolve(action, current_action->second,
symbol, item_set);
const ParseAction current_action = current_entry->second[0];
auto resolution = conflict_manager.resolve(action, current_action, lookahead_sym);
switch (resolution.second) {
case ConflictTypeNone:
if (resolution.first)
return &parse_table.set_action(state_id, lookahead_sym, action);
break;
switch (get<1>(result)) {
case ConflictTypeResolved:
if (action.type == ParseActionTypeReduce)
parse_table.fragile_production_ids.insert(action.production_id);
if (current_action->second.type == ParseActionTypeReduce)
parse_table.fragile_production_ids.insert(current_action->second.production_id);
if (current_action.type == ParseActionTypeReduce)
parse_table.fragile_production_ids.insert(current_action.production_id);
if (resolution.first)
return &parse_table.set_action(state_id, lookahead_sym, action);
break;
case ConflictTypeError:
conflicts.insert(get<2>(result));
break;
default:
case ConflictTypeUnresolved: {
set<Symbol> goal_symbols = item_set_goal_symbols(item_set);
if (has_expected_conflict(goal_symbols))
return &parse_table.add_action(state_id, lookahead_sym, action);
else
conflicts.insert(conflict_description(action, current_action, lookahead_sym, goal_symbols));
break;
}
}
return get<0>(result);
return nullptr;
}
bool has_expected_conflict(const set<Symbol> &symbols) {
for (const auto &conflicting_symbols : grammar.expected_conflicts)
if (symbols == conflicting_symbols)
return true;
return false;
}
set<int> precedence_values_for_item_set(const ParseItemSet &item_set) {
@ -191,6 +203,93 @@ class ParseTableBuilder {
}
return result;
}
set<Symbol> item_set_goal_symbols(const ParseItemSet &item_set) {
set<Symbol> result;
for (const auto &pair : item_set) {
const ParseItem &item = pair.first;
if (!item.consumed_symbols.empty())
result.insert(item.lhs);
}
return result;
}
string conflict_description(const ParseAction &new_action,
const ParseAction &old_action,
const rules::Symbol &symbol,
const set<Symbol> &goal_symbols) const {
string symbols_string;
bool started = false;
for (const auto &symbol : goal_symbols) {
if (started)
symbols_string += ", ";
symbols_string += symbol_name(symbol);
started = true;
}
return
"Within: " + symbols_string + "\n"
"Lookahead: " + symbol_name(symbol) + "\n" +
"Possible Actions:\n"
"* " + action_description(old_action) + "\n" +
"* " + action_description(new_action);
}
string symbol_name(const rules::Symbol &symbol) const {
if (symbol.is_built_in()) {
if (symbol == rules::ERROR())
return "ERROR";
else if (symbol == rules::END_OF_INPUT())
return "END_OF_INPUT";
else
return "";
} else if (symbol.is_token())
return lexical_grammar.rule_name(symbol);
else
return grammar.rule_name(symbol);
}
string action_description(const ParseAction &action) const {
string result;
switch (action.type) {
case ParseActionTypeReduce: {
result = "Reduce";
for (const rules::Symbol &symbol : productions[action.production_id])
result += " " + symbol_name(symbol);
result += " -> " + symbol_name(action.symbol);
break;
}
case ParseActionTypeShift: {
result = "Shift";
break;
}
default:
return "";
}
if (action.precedence_values.size() > 1) {
result += " (Precedences " + to_string(*action.precedence_values.begin()) +
", " + to_string(*action.precedence_values.rbegin()) + ")";
} else {
result += " (Precedence " + to_string(*action.precedence_values.begin()) + ")";
}
return result;
}
size_t get_production_id(const vector<rules::Symbol> &symbols) {
auto begin = productions.begin();
auto end = productions.end();
auto iter = find(begin, end, symbols);
if (iter == end) {
productions.push_back(symbols);
return productions.size() - 1;
}
return iter - begin;
}
};
pair<ParseTable, const GrammarError *> build_parse_table(

125
src/compiler/build_tables/parse_conflict_manager.cc
View file

@ -8,31 +8,26 @@ namespace tree_sitter {
namespace build_tables {
using std::find;
using std::get;
using std::make_tuple;
using std::string;
using std::to_string;
using std::tuple;
using std::pair;
using std::vector;
ParseConflictManager::ParseConflictManager(const SyntaxGrammar &syntax_grammar,
const LexicalGrammar &lexical_grammar) :
syntax_grammar(syntax_grammar),
lexical_grammar(lexical_grammar) {}
ParseConflictManager::ParseConflictManager(const SyntaxGrammar &syntax_grammar)
: syntax_grammar(syntax_grammar) {}
tuple<bool, ConflictType, string>
pair<bool, ConflictType>
ParseConflictManager::resolve(const ParseAction &new_action,
const ParseAction &old_action,
const rules::Symbol &symbol,
const ParseItemSet &item_set) const {
const rules::Symbol &symbol) const {
if (new_action.type < old_action.type) {
auto opposite = resolve(old_action, new_action, symbol, item_set);
return make_tuple(!get<0>(opposite), get<1>(opposite), get<2>(opposite));
auto opposite = resolve(old_action, new_action, symbol);
return {!opposite.first, opposite.second};
}
switch (old_action.type) {
case ParseActionTypeError:
return make_tuple(true, ConflictTypeNone, "");
case ParseActionTypeShiftExtra:
case ParseActionTypeReduceExtra:
return {true, ConflictTypeNone};
case ParseActionTypeShift:
if (new_action.type == ParseActionTypeReduce) {
@ -40,20 +35,20 @@ ParseConflictManager::resolve(const ParseAction &new_action,
int max_precedence = *old_action.precedence_values.rbegin();
int new_precedence = *new_action.precedence_values.rbegin();
if (new_precedence < min_precedence)
return make_tuple(false, ConflictTypeResolved, "");
return {false, ConflictTypeResolved};
else if (new_precedence > max_precedence)
return make_tuple(true, ConflictTypeResolved, "");
return {true, ConflictTypeResolved};
else if (min_precedence == max_precedence) {
switch (new_action.associativity) {
case rules::AssociativityLeft:
return make_tuple(true, ConflictTypeResolved, "");
return {true, ConflictTypeResolved};
case rules::AssociativityRight:
return make_tuple(false, ConflictTypeResolved, "");
return {false, ConflictTypeResolved};
default:
return make_tuple(false, ConflictTypeError, conflict_description(new_action, old_action, symbol, item_set));
return {false, ConflictTypeUnresolved};
}
} else {
return make_tuple(false, ConflictTypeError, conflict_description(new_action, old_action, symbol, item_set));
return {false, ConflictTypeUnresolved};
}
}
@ -62,99 +57,19 @@ ParseConflictManager::resolve(const ParseAction &new_action,
int old_precedence = *old_action.precedence_values.begin();
int new_precedence = *new_action.precedence_values.begin();
if (new_precedence > old_precedence) {
return make_tuple(true, ConflictTypeResolved, "");
return {true, ConflictTypeResolved};
} else if (new_precedence < old_precedence) {
return make_tuple(false, ConflictTypeResolved, "");
return {false, ConflictTypeResolved};
} else {
return make_tuple(false, ConflictTypeError, conflict_description(new_action, old_action, symbol, item_set));
return {false, ConflictTypeUnresolved};
}
}
default:
return make_tuple(false, ConflictTypeNone, "");
}
}
size_t ParseConflictManager::get_production_id(const vector<rules::Symbol> &symbols) {
auto begin = productions.begin();
auto end = productions.end();
auto iter = find(begin, end, symbols);
if (iter == end) {
productions.push_back(symbols);
return productions.size() - 1;
}
return iter - begin;
}
string ParseConflictManager::item_set_description(const ParseItemSet &item_set) const {
string result;
bool started = false;
for (const auto &pair : item_set) {
const ParseItem &item = pair.first;
if (!item.consumed_symbols.empty()) {
if (started) result += ", ";
result += symbol_name(item.lhs);
started = true;
}
}
return result;
}
string ParseConflictManager::conflict_description(const ParseAction &new_action,
const ParseAction &old_action,
const rules::Symbol &symbol,
const ParseItemSet &item_set) const {
return
"Within: " + item_set_description(item_set) + "\n"
"Lookahead: " + symbol_name(symbol) + "\n" +
"Possible Actions:\n"
"* " + action_description(old_action) + "\n" +
"* " + action_description(new_action);
}
string ParseConflictManager::symbol_name(const rules::Symbol &symbol) const {
if (symbol.is_built_in()) {
if (symbol == rules::ERROR())
return "ERROR";
else if (symbol == rules::END_OF_INPUT())
return "END_OF_INPUT";
else
return "";
} else if (symbol.is_token())
return lexical_grammar.rule_name(symbol);
else
return syntax_grammar.rule_name(symbol);
}
string ParseConflictManager::action_description(const ParseAction &action) const {
string result;
switch (action.type) {
case ParseActionTypeReduce: {
result = "Reduce";
for (const rules::Symbol &symbol : productions[action.production_id])
result += " " + symbol_name(symbol);
result += " -> " + symbol_name(action.symbol);
break;
}
case ParseActionTypeShift: {
result = "Shift";
break;
}
default:
return "";
}
if (action.precedence_values.size() > 1) {
result += " (Precedences " + to_string(*action.precedence_values.begin()) +
", " + to_string(*action.precedence_values.rbegin()) + ")";
} else {
result += " (Precedence " + to_string(*action.precedence_values.begin()) + ")";
}
return result;
return {false, ConflictTypeNone};
}
} // namespace build_tables

17
src/compiler/build_tables/parse_conflict_manager.h
View file

@ -17,25 +17,16 @@ namespace build_tables {
enum ConflictType {
ConflictTypeNone,
ConflictTypeResolved,
ConflictTypeError
ConflictTypeUnresolved
};
class ParseConflictManager {
const SyntaxGrammar syntax_grammar;
const LexicalGrammar lexical_grammar;
std::vector<std::vector<rules::Symbol>> productions;
public:
ParseConflictManager(const SyntaxGrammar &, const LexicalGrammar &);
size_t get_production_id(const std::vector<rules::Symbol> &);
std::tuple<bool, ConflictType, std::string> resolve(
const ParseAction &, const ParseAction &, const rules::Symbol &, const ParseItemSet &) const;
private:
std::string symbol_name(const rules::Symbol &) const;
std::string item_set_description(const ParseItemSet &) const;
std::string action_description(const ParseAction &) const;
std::string conflict_description(const ParseAction &, const ParseAction &, const rules::Symbol &, const ParseItemSet &) const;
ParseConflictManager(const SyntaxGrammar &);
std::pair<bool, ConflictType> resolve(
const ParseAction &, const ParseAction &, const rules::Symbol &) const;
};
} // namespace build_tables

54
src/compiler/generate_code/c_code.cc
View file

@ -192,7 +192,7 @@ class CCodeGenerator {
for (const auto &pair : state.actions) {
line("[" + symbol_id(pair.first) + "] = ");
add("ACTIONS(");
add_parse_action(pair.second);
add_parse_actions(pair.second);
add("),");
}
});
@ -279,29 +279,35 @@ class CCodeGenerator {
}
}
void add_parse_action(const ParseAction &action) {
switch (action.type) {
case ParseActionTypeAccept:
add("ACCEPT_INPUT()");
break;
case ParseActionTypeShift:
add("SHIFT(" + to_string(action.state_index) + ")");
break;
case ParseActionTypeShiftExtra:
add("SHIFT_EXTRA()");
break;
case ParseActionTypeReduce:
if (reduce_action_is_fragile(action))
add("REDUCE_FRAGILE(" + symbol_id(action.symbol) + ", " +
to_string(action.consumed_symbol_count) + ")");
else
add("REDUCE(" + symbol_id(action.symbol) + ", " +
to_string(action.consumed_symbol_count) + ")");
break;
case ParseActionTypeReduceExtra:
add("REDUCE_EXTRA(" + symbol_id(action.symbol) + ")");
break;
default: {}
void add_parse_actions(const vector<ParseAction> &actions) {
bool started = false;
for (const auto &action : actions) {
if (started)
add(", ");
switch (action.type) {
case ParseActionTypeAccept:
add("ACCEPT_INPUT()");
break;
case ParseActionTypeShift:
add("SHIFT(" + to_string(action.state_index) + ")");
break;
case ParseActionTypeShiftExtra:
add("SHIFT_EXTRA()");
break;
case ParseActionTypeReduce:
if (reduce_action_is_fragile(action))
add("REDUCE_FRAGILE(" + symbol_id(action.symbol) + ", " +
to_string(action.consumed_symbol_count) + ")");
else
add("REDUCE(" + symbol_id(action.symbol) + ", " +
to_string(action.consumed_symbol_count) + ")");
break;
case ParseActionTypeReduceExtra:
add("REDUCE_EXTRA(" + symbol_id(action.symbol) + ")");
break;
default: {}
}
started = true;
}
}

17
src/compiler/parse_table.cc
View file

@ -122,7 +122,11 @@ ostream &operator<<(ostream &stream, const ParseState &state) {
for (auto pair : state.actions) {
if (started)
stream << string(", ");
stream << pair.first << string(" => ") << pair.second;
stream << pair.first << string(" => {");
for (auto &action : pair.second) {
stream << string(" ") << action;
}
stream << string("}");
started = true;
}
stream << string(">");
@ -134,9 +138,16 @@ ParseStateId ParseTable::add_state() {
return states.size() - 1;
}
void ParseTable::add_action(ParseStateId id, Symbol symbol, ParseAction action) {
ParseAction &ParseTable::set_action(ParseStateId id, Symbol symbol, ParseAction action) {
symbols.insert(symbol);
states[id].actions[symbol] = action;
states[id].actions[symbol] = vector<ParseAction>({ action });
return *states[id].actions[symbol].begin();
}
ParseAction &ParseTable::add_action(ParseStateId id, Symbol symbol, ParseAction action) {
symbols.insert(symbol);
states[id].actions[symbol].push_back(action);
return *states[id].actions[symbol].rbegin();
}
} // namespace tree_sitter

13
src/compiler/parse_table.h
View file

@ -15,10 +15,11 @@ typedef uint64_t ParseStateId;
typedef enum {
ParseActionTypeError,
ParseActionTypeShift,
ParseActionTypeShiftExtra,
ParseActionTypeReduce,
ParseActionTypeReduceExtra,
ParseActionTypeShiftExtra,
ParseActionTypeShift,
ParseActionTypeReduce,
ParseActionTypeAccept,
} ParseActionType;
@ -74,7 +75,7 @@ namespace tree_sitter {
class ParseState {
public:
ParseState();
std::map<rules::Symbol, ParseAction> actions;
std::map<rules::Symbol, std::vector<ParseAction>> actions;
std::set<rules::Symbol> expected_inputs() const;
LexStateId lex_state_id;
};
@ -84,7 +85,9 @@ std::ostream &operator<<(std::ostream &stream, const ParseState &state);
class ParseTable {
public:
ParseStateId add_state();
void add_action(ParseStateId state_id, rules::Symbol symbol,
ParseAction &set_action(ParseStateId state_id, rules::Symbol symbol,
ParseAction action);
ParseAction &add_action(ParseStateId state_id, rules::Symbol symbol,
ParseAction action);
std::vector<ParseState> states;