Respect expected_conflicts field when building parse table

This commit is contained in:
Max Brunsfeld 2015-06-28 16:22:31 -05:00
parent c9a482bbf3
commit aabcb10cfb
15 changed files with 5054 additions and 348 deletions

View file

@ -25,18 +25,18 @@ describe("build_parse_table", []() {
it("first looks for the start rule and its item set closure", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
AssertThat(result.first.states[0].actions, Equals(map<Symbol, ParseAction>({
AssertThat(result.first.states[0].actions, Equals(map<Symbol, vector<ParseAction>>({
// start item
{ Symbol(0), ParseAction::Shift(1, { 0 }) },
{ Symbol(0), {ParseAction::Shift(1, { 0 })} },
// expanded from the item set closure of the start item
{ Symbol(1), ParseAction::Shift(2, { 0 }) },
{ Symbol(2), ParseAction::Shift(2, { 0 }) },
{ Symbol(0, SymbolOptionToken), ParseAction::Shift(3, { 0 }) },
{ Symbol(1, SymbolOptionToken), ParseAction::Shift(4, { 0 }) },
{ Symbol(1), {ParseAction::Shift(2, { 0 })} },
{ Symbol(2), {ParseAction::Shift(2, { 0 })} },
{ Symbol(0, SymbolOptionToken), {ParseAction::Shift(3, { 0 })} },
{ Symbol(1, SymbolOptionToken), {ParseAction::Shift(4, { 0 })} },
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
{ Symbol(2, SymbolOptionToken), {ParseAction::ShiftExtra()} },
})));
});
@ -48,22 +48,22 @@ describe("build_parse_table", []() {
it("accepts the input when EOF occurs after the start rule", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
AssertThat(result.first.states[1].actions, Equals(map<Symbol, ParseAction>({
{ END_OF_INPUT(), ParseAction::Accept() },
AssertThat(result.first.states[1].actions, Equals(map<Symbol, vector<ParseAction>>({
{ END_OF_INPUT(), {ParseAction::Accept()} },
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
{ Symbol(2, SymbolOptionToken), {ParseAction::ShiftExtra()} },
})));
});
it("reduces a rule once it has been consumed", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
AssertThat(result.first.states[2].actions, Equals(map<Symbol, ParseAction>({
{ END_OF_INPUT(), ParseAction::Reduce(Symbol(0), 1, 0, AssociativityLeft, 0) },
AssertThat(result.first.states[2].actions, Equals(map<Symbol, vector<ParseAction>>({
{ END_OF_INPUT(), {ParseAction::Reduce(Symbol(0), 1, 0, AssociativityLeft, 0)} },
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
{ Symbol(2, SymbolOptionToken), {ParseAction::ShiftExtra()} },
})));
});
});

View file

@ -18,59 +18,48 @@ describe("ParseConflictManager", []() {
{ "other_rule2", i_token(0) },
}, {}, { Symbol(2, SymbolOptionToken) }, set<set<Symbol>>());
LexicalGrammar lexical_grammar({
{ "other_token", pattern("[a-b]") },
{ "lookahead_token", pattern("[c-d]") },
}, {});
tuple<bool, ConflictType, string> result;
pair<bool, ConflictType> result;
Symbol sym1(0);
Symbol sym2(1);
Symbol lookahead_sym(1, SymbolOptionToken);
ParseConflictManager *conflict_manager;
before_each([&]() {
conflict_manager = new ParseConflictManager(syntax_grammar, lexical_grammar);
conflict_manager = new ParseConflictManager(syntax_grammar);
});
after_each([&]() {
delete conflict_manager;
});
describe(".get_production_id", [&]() {
it("returns different IDs for different productions", [&]() {
int id1 = conflict_manager->get_production_id(vector<Symbol>({ Symbol(1), Symbol(2) }));
AssertThat(id1, Equals(0));
int id2 = conflict_manager->get_production_id(vector<Symbol>({ Symbol(1), Symbol(2), Symbol(3) }));
AssertThat(id2, Equals(1));
int id3 = conflict_manager->get_production_id(vector<Symbol>({ Symbol(1) }));
AssertThat(id3, Equals(2));
int id4 = conflict_manager->get_production_id(vector<Symbol>({ Symbol(1), Symbol(2) }));
AssertThat(id4, Equals(id1));
});
});
describe(".resolve", [&]() {
ParseItemSet item_set({
{ ParseItem(Symbol(0), blank(), { Symbol(0, SymbolOptionToken) }), set<Symbol>() }, // in_progress_rule1
{ ParseItem(Symbol(3), blank(), {}), set<Symbol>() }, // other_rule1
});
describe("errors", [&]() {
ParseAction error = ParseAction::Error();
ParseAction non_error = ParseAction::Shift(2, { 0 });
it("favors non-errors and reports no conflict", [&]() {
result = conflict_manager->resolve(non_error, error, sym1, item_set);
AssertThat(get<0>(result), IsTrue());
AssertThat(get<1>(result), Equals(ConflictTypeNone));
result = conflict_manager->resolve(non_error, error, sym1);
AssertThat(result.first, IsTrue());
AssertThat(result.second, Equals(ConflictTypeNone));
result = conflict_manager->resolve(error, non_error, sym1, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeNone));
result = conflict_manager->resolve(error, non_error, sym1);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeNone));
});
});
describe("shift-extra actions", [&]() {
ParseAction shift_extra = ParseAction::Error();
ParseAction other = ParseAction::Shift(2, { 0 });
it("favors other actions over shift-extra actions", [&]() {
result = conflict_manager->resolve(other, shift_extra, sym1);
AssertThat(result.first, IsTrue());
AssertThat(result.second, Equals(ConflictTypeNone));
result = conflict_manager->resolve(shift_extra, other, sym1);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeNone));
});
});
@ -80,13 +69,13 @@ describe("ParseConflictManager", []() {
ParseAction reduce = ParseAction::Reduce(sym2, 1, 1, AssociativityLeft, 0);
it("favors the shift and reports the conflict as resolved", [&]() {
result = conflict_manager->resolve(shift, reduce, sym1, item_set);
AssertThat(get<0>(result), IsTrue());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(shift, reduce, sym1);
AssertThat(result.first, IsTrue());
AssertThat(result.second, Equals(ConflictTypeResolved));
result = conflict_manager->resolve(reduce, shift, sym1, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(reduce, shift, sym1);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeResolved));
});
});
@ -97,13 +86,13 @@ describe("ParseConflictManager", []() {
ParseAction reduce = ParseAction::Reduce(sym2, 1, 3, AssociativityLeft, 0);
it("favors the reduce and reports the conflict as resolved", [&]() {
result = conflict_manager->resolve(shift, reduce, sym1, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(shift, reduce, sym1);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeResolved));
result = conflict_manager->resolve(reduce, shift, sym1, item_set);
AssertThat(get<0>(result), IsTrue());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(reduce, shift, sym1);
AssertThat(result.first, IsTrue());
AssertThat(result.second, Equals(ConflictTypeResolved));
});
});
@ -112,13 +101,13 @@ describe("ParseConflictManager", []() {
ParseAction reduce = ParseAction::Reduce(sym2, 1, 0, AssociativityLeft, 0);
it("favors the reduce and reports the conflict as resolved", [&]() {
result = conflict_manager->resolve(reduce, shift, sym1, item_set);
AssertThat(get<0>(result), IsTrue());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(reduce, shift, sym1);
AssertThat(result.first, IsTrue());
AssertThat(result.second, Equals(ConflictTypeResolved));
result = conflict_manager->resolve(shift, reduce, sym1, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(shift, reduce, sym1);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeResolved));
});
});
@ -127,13 +116,13 @@ describe("ParseConflictManager", []() {
ParseAction reduce = ParseAction::Reduce(sym2, 1, 0, AssociativityRight, 0);
it("favors the shift, and reports the conflict as resolved", [&]() {
result = conflict_manager->resolve(reduce, shift, sym1, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(reduce, shift, sym1);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeResolved));
result = conflict_manager->resolve(shift, reduce, sym1, item_set);
AssertThat(get<0>(result), IsTrue());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(shift, reduce, sym1);
AssertThat(result.first, IsTrue());
AssertThat(result.second, Equals(ConflictTypeResolved));
});
});
@ -142,31 +131,12 @@ describe("ParseConflictManager", []() {
ParseAction shift = ParseAction::Shift(2, { 0 });
ParseAction reduce = ParseAction::Reduce(Symbol(2), 1, 0, AssociativityUnspecified, 0);
reduce.production_id = conflict_manager->get_production_id(vector<Symbol>({
Symbol(3),
Symbol(4),
}));
result = conflict_manager->resolve(reduce, shift, lookahead_sym);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeUnresolved));
result = conflict_manager->resolve(reduce, shift, lookahead_sym, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeError));
AssertThat(get<2>(result), Equals(
"Within: in_progress_rule1\n"
"Lookahead: lookahead_token\n"
"Possible Actions:\n"
"* Shift (Precedence 0)\n"
"* Reduce other_rule1 other_rule2 -> reduced_rule (Precedence 0)"
));
result = conflict_manager->resolve(shift, reduce, lookahead_sym, item_set);
AssertThat(get<0>(result), IsTrue());
AssertThat(get<2>(result), Equals(
"Within: in_progress_rule1\n"
"Lookahead: lookahead_token\n"
"Possible Actions:\n"
"* Shift (Precedence 0)\n"
"* Reduce other_rule1 other_rule2 -> reduced_rule (Precedence 0)"
));
result = conflict_manager->resolve(shift, reduce, lookahead_sym);
AssertThat(result.first, IsTrue());
});
});
@ -175,25 +145,13 @@ describe("ParseConflictManager", []() {
ParseAction reduce = ParseAction::Reduce(Symbol(2), 1, 2, AssociativityLeft, 0);
it("returns false and reports an unresolved conflict", [&]() {
reduce.production_id = conflict_manager->get_production_id(vector<Symbol>({
Symbol(3),
Symbol(4),
}));
result = conflict_manager->resolve(reduce, shift, lookahead_sym);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeUnresolved));
result = conflict_manager->resolve(reduce, shift, lookahead_sym, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeError));
result = conflict_manager->resolve(shift, reduce, lookahead_sym, item_set);
AssertThat(get<0>(result), IsTrue());
AssertThat(get<1>(result), Equals(ConflictTypeError));
AssertThat(get<2>(result), Equals(
"Within: in_progress_rule1\n"
"Lookahead: lookahead_token\n"
"Possible Actions:\n"
"* Shift (Precedences 0, 3)\n"
"* Reduce other_rule1 other_rule2 -> reduced_rule (Precedence 2)"
));
result = conflict_manager->resolve(shift, reduce, lookahead_sym);
AssertThat(result.first, IsTrue());
AssertThat(result.second, Equals(ConflictTypeUnresolved));
});
});
});
@ -204,13 +162,13 @@ describe("ParseConflictManager", []() {
ParseAction right = ParseAction::Reduce(sym2, 1, 3, AssociativityLeft, 0);
it("favors that action", [&]() {
result = conflict_manager->resolve(left, right, sym1, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(left, right, sym1);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeResolved));
result = conflict_manager->resolve(right, left, sym1, item_set);
AssertThat(get<0>(result), IsTrue());
AssertThat(get<1>(result), Equals(ConflictTypeResolved));
result = conflict_manager->resolve(right, left, sym1);
AssertThat(result.first, IsTrue());
AssertThat(result.second, Equals(ConflictTypeResolved));
});
});
@ -219,36 +177,13 @@ describe("ParseConflictManager", []() {
ParseAction left = ParseAction::Reduce(Symbol(2), 1, 0, AssociativityLeft, 0);
ParseAction right = ParseAction::Reduce(Symbol(3), 1, 0, AssociativityLeft, 0);
left.production_id = conflict_manager->get_production_id(vector<Symbol>({
Symbol(3),
Symbol(4),
}));
result = conflict_manager->resolve(right, left, lookahead_sym);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeUnresolved));
right.production_id = conflict_manager->get_production_id(vector<Symbol>({
Symbol(4),
}));
result = conflict_manager->resolve(right, left, lookahead_sym, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeError));
AssertThat(get<2>(result), Equals(
"Within: in_progress_rule1\n"
"Lookahead: lookahead_token\n"
"Possible Actions:\n"
"* Reduce other_rule1 other_rule2 -> reduced_rule (Precedence 0)\n"
"* Reduce other_rule2 -> other_rule1 (Precedence 0)"
));
result = conflict_manager->resolve(left, right, lookahead_sym, item_set);
AssertThat(get<0>(result), IsFalse());
AssertThat(get<1>(result), Equals(ConflictTypeError));
AssertThat(get<2>(result), Equals(
"Within: in_progress_rule1\n"
"Lookahead: lookahead_token\n"
"Possible Actions:\n"
"* Reduce other_rule2 -> other_rule1 (Precedence 0)\n"
"* Reduce other_rule1 other_rule2 -> reduced_rule (Precedence 0)"
));
result = conflict_manager->resolve(left, right, lookahead_sym);
AssertThat(result.first, IsFalse());
AssertThat(result.second, Equals(ConflictTypeUnresolved));
});
});
});

View file

@ -14,6 +14,7 @@ extern const Grammar arithmetic;
extern const Grammar javascript;
extern const Grammar json;
extern const Grammar golang;
extern const Grammar c;
} // namespace tree_sitter_examples
@ -40,6 +41,7 @@ describe("compiling the example grammars", []() {
compile_grammar(tree_sitter_examples::json, "json");
compile_grammar(tree_sitter_examples::javascript, "javascript");
compile_grammar(tree_sitter_examples::golang, "golang");
compile_grammar(tree_sitter_examples::c, "c");
});
END_TEST

164
spec/fixtures/grammars/c.cc vendored Normal file
View file

@ -0,0 +1,164 @@
#include "tree_sitter/compiler.h"
#include "helpers.h"
namespace tree_sitter_examples {
using tree_sitter::Grammar;
using namespace tree_sitter::rules;
// http://slps.github.io/zoo/c/iso-9899-tc3.html
extern const Grammar c = Grammar({
{ "program", choice({
sym("function_definition"),
sym("declaration") }) },
{ "function_definition", seq({
optional(sym("declaration_specifiers")),
sym("type_specifier"),
sym("declarator"),
repeat(sym("declaration")),
sym("compound_statement") }) },
{ "declaration_specifiers", repeat1(choice({
sym("storage_class_specifier"),
sym("type_qualifier") })) },
{ "storage_class_specifier", choice({
str("typedef"),
str("extern"),
str("static"),
str("auto"),
str("register") }) },
{ "type_specifier", choice({
sym("struct_specifier"),
seq({
repeat(choice({
str("signed"),
str("unsigned"),
str("long"),
str("short") })),
sym("identifier") }) }) },
{ "struct_specifier", seq({
str("struct"),
optional(sym("identifier")),
seq({
str("{"),
repeat(sym("struct_declaration")),
str("}") }) }) },
{ "struct_declaration", seq({
sym("type_specifier"),
sym("declarator") }) },
{ "parameter_declaration", seq({
optional(sym("declaration_specifiers")),
sym("type_specifier"),
sym("declarator") }) },
{ "declaration", seq({
optional(sym("declaration_specifiers")),
sym("type_specifier"),
comma_sep1(sym("init_declarator")),
str(";") }) },
{ "init_declarator", choice({
sym("declarator"),
seq({ sym("declarator"), str("="), sym("initializer") }) }) },
{ "initializer", choice({
sym("expression"),
seq({
str("{"),
sym("initializer_list"),
optional(str(",")),
str("}") }) }) },
{ "initializer_list", choice({
seq({
optional(sym("designation")),
sym("initializer") }),
seq({
sym("initializer_list"),
str(","),
optional(sym("designation")),
sym("initializer") }) }) },
{ "designation", seq({
repeat1(choice({
seq({
str("["),
sym("expression"),
str("]") }),
seq({
str("."),
sym("identifier") }) })),
str("=") }) },
{ "declarator", seq({
repeat(sym("star")),
sym("direct_declarator") }) },
{ "direct_declarator", choice({
sym("identifier"),
seq({
str("("),
sym("declarator"),
str(")") }),
seq({
sym("direct_declarator"),
str("["),
optional(sym("expression")),
str("]") }),
seq({
sym("direct_declarator"),
str("("),
comma_sep(sym("parameter_declaration")),
str(")") }) }) },
{ "type_qualifier", choice({
str("const"),
str("restrict"),
str("volatile") }) },
{ "star", str("*") },
{ "compound_statement", seq({
str("{"),
repeat(choice({ sym("declaration"), sym("statement") })),
str("}") }) },
{ "expression", choice({
sym("math_expression"),
sym("string"),
sym("identifier"),
sym("number") }) },
{ "math_expression", choice({
prec(1, seq({ sym("expression"), str("+"), sym("expression") })),
prec(2, seq({ sym("expression"), sym("star"), sym("expression") })) }) },
{ "statement", choice({
sym("expression_statement") }) },
{ "expression_statement", seq({
sym("expression"),
str(";") }) },
{ "string", delimited("\"") },
{ "identifier", pattern("\\a[\\w_]*") },
{ "number", pattern("\\d+(\\.\\d+)?") },
{ "comment", pattern("//[^\n]*") },
}).ubiquitous_tokens({
sym("comment"),
pattern("[ \t\r\n]"),
}).expected_conflicts({
{ "type_specifier", "expression" },
});
} // namespace tree_sitter_examples

View file

@ -4,6 +4,10 @@ namespace tree_sitter_examples {
using namespace tree_sitter::rules;
rule_ptr repeat1(rule_ptr element) {
return seq({ element, repeat(element) });
}
rule_ptr comma_sep1(rule_ptr element) {
return seq({ element, repeat(seq({ str(","), element })) });
}

View file

@ -7,6 +7,7 @@ namespace tree_sitter_examples {
using namespace tree_sitter::rules;
rule_ptr repeat1(rule_ptr element);
rule_ptr comma_sep1(rule_ptr element);
rule_ptr comma_sep(rule_ptr element);
rule_ptr optional(rule_ptr rule);

4535
spec/fixtures/parsers/c.c vendored Normal file

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,38 @@
==========================================
simple declarations
==========================================
int x;
---
(declaration (identifier) (identifier))
==========================================
simple functions
==========================================
int main() {
}
---
(function_definition
(identifier) (direct_declarator (identifier))
(compound_statement))
==========================================
simple declarations in functions
==========================================
int main() {
x + y;
}
---
(function_definition
(identifier)
(direct_declarator (identifier))
(compound_statement
(expression_statement (math_expression (identifier) (identifier)))))

View file

@ -8,6 +8,7 @@ extern "C" const TSLanguage *ts_language_javascript();
extern "C" const TSLanguage *ts_language_json();
extern "C" const TSLanguage *ts_language_arithmetic();
extern "C" const TSLanguage *ts_language_golang();
extern "C" const TSLanguage *ts_language_c();
START_TEST
@ -23,7 +24,7 @@ describe("Languages", [&]() {
});
auto run_tests_for_language = [&](string language_name, const TSLanguage *language) {
describe(language_name.c_str(), [&]() {
describe((string("The ") + language_name + " parser").c_str(), [&]() {
before_each([&]() {
ts_document_set_language(doc, language);
// ts_document_set_debugger(doc, log_debugger_make());
@ -85,6 +86,7 @@ describe("Languages", [&]() {
run_tests_for_language("arithmetic", ts_language_arithmetic());
run_tests_for_language("javascript", ts_language_javascript());
run_tests_for_language("golang", ts_language_golang());
run_tests_for_language("c", ts_language_c());
});
END_TEST