traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix indentation in specs

Browse source
This commit is contained in:
Max Brunsfeld 2014-08-06 13:00:35 -07:00
parent 01571da30d
commit b155994491
34 changed files with 1877 additions and 1878 deletions
Download patch file Download diff file Expand all files Collapse all files

86
spec/compiler/build_tables/build_parse_table_spec.cc
View file

@ -10,61 +10,61 @@ using namespace build_tables;
START_TEST
describe("building parse tables", []() {
SyntaxGrammar parse_grammar({
{ "rule0", choice({ i_sym(1), i_sym(2) }) },
{ "rule1", i_token(0) },
{ "rule2", i_token(1) },
}, {}, { Symbol(2, SymbolOptionToken) });
SyntaxGrammar parse_grammar({
{ "rule0", choice({ i_sym(1), i_sym(2) }) },
{ "rule1", i_token(0) },
{ "rule2", i_token(1) },
}, {}, { Symbol(2, SymbolOptionToken) });
LexicalGrammar lex_grammar({
{ "token0", pattern("[a-c]") },
{ "token1", pattern("[b-d]") },
}, {});
LexicalGrammar lex_grammar({
{ "token0", pattern("[a-c]") },
{ "token1", pattern("[b-d]") },
}, {});
it("first looks for the start rule and its item set closure", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
it("first looks for the start rule and its item set closure", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
AssertThat(result.first.states[0].actions, Equals(map<Symbol, ParseAction>({
// start item
{ Symbol(0), ParseAction::Shift(1, { 0 }) },
AssertThat(result.first.states[0].actions, Equals(map<Symbol, ParseAction>({
// start item
{ Symbol(0), ParseAction::Shift(1, { 0 }) },
// expanded from the item set closure of the start item
{ Symbol(1), ParseAction::Shift(2, { 0 }) },
{ Symbol(2), ParseAction::Shift(2, { 0 }) },
{ Symbol(0, SymbolOptionToken), ParseAction::Shift(3, { 0 }) },
{ Symbol(1, SymbolOptionToken), ParseAction::Shift(4, { 0 }) },
// expanded from the item set closure of the start item
{ Symbol(1), ParseAction::Shift(2, { 0 }) },
{ Symbol(2), ParseAction::Shift(2, { 0 }) },
{ Symbol(0, SymbolOptionToken), ParseAction::Shift(3, { 0 }) },
{ Symbol(1, SymbolOptionToken), ParseAction::Shift(4, { 0 }) },
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
})));
});
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
})));
});
it("ensures that the built-in 'ERROR' symbol is in the symbol set", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
AssertThat(result.first.symbols, Contains(rules::ERROR()));
});
it("ensures that the built-in 'ERROR' symbol is in the symbol set", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
AssertThat(result.first.symbols, Contains(rules::ERROR()));
});
it("accepts the input when EOF occurs after the start rule", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
it("accepts the input when EOF occurs after the start rule", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
AssertThat(result.first.states[1].actions, Equals(map<Symbol, ParseAction>({
{ END_OF_INPUT(), ParseAction::Accept() },
AssertThat(result.first.states[1].actions, Equals(map<Symbol, ParseAction>({
{ END_OF_INPUT(), ParseAction::Accept() },
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
})));
});
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
})));
});
it("reduces a rule once it has been consumed", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
it("reduces a rule once it has been consumed", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
AssertThat(result.first.states[2].actions, Equals(map<Symbol, ParseAction>({
{ END_OF_INPUT(), ParseAction::Reduce(Symbol(0), 1, 0) },
AssertThat(result.first.states[2].actions, Equals(map<Symbol, ParseAction>({
{ END_OF_INPUT(), ParseAction::Reduce(Symbol(0), 1, 0) },
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
})));
});
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
})));
});
});

406
spec/compiler/build_tables/conflict_manager_spec.cc
View file

@ -10,217 +10,217 @@ using namespace build_tables;
START_TEST
describe("resolving parse conflicts", []() {
bool update;
bool update;
SyntaxGrammar parse_grammar({
{ "rule1", seq({ sym("rule2"), sym("token2") }) },
{ "rule2", sym("token1") },
}, {}, set<rules::Symbol>());
SyntaxGrammar parse_grammar({
{ "rule1", seq({ sym("rule2"), sym("token2") }) },
{ "rule2", sym("token1") },
}, {}, set<rules::Symbol>());
LexicalGrammar lex_grammar({
{ "token1", pattern("[a-c]") },
{ "token2", pattern("[b-d]") },
{ "token3", keyword("stuff") },
}, {}, set<char>());
LexicalGrammar lex_grammar({
{ "token1", pattern("[a-c]") },
{ "token2", pattern("[b-d]") },
{ "token3", keyword("stuff") },
}, {}, set<char>());
describe("lexical conflicts", [&]() {
Symbol sym1(0, SymbolOptionToken);
Symbol sym2(1, SymbolOptionToken);
Symbol sym3(2, SymbolOptionToken);
describe("lexical conflicts", [&]() {
Symbol sym1(0, SymbolOptionToken);
Symbol sym2(1, SymbolOptionToken);
Symbol sym3(2, SymbolOptionToken);
LexConflictManager *manager;
LexConflictManager *manager;
before_each([&]() {
manager = new LexConflictManager(lex_grammar);
});
after_each([&]() {
delete manager;
});
it("favors non-errors over lexical errors", [&]() {
update = manager->resolve_lex_action(LexAction::Error(), LexAction::Advance(2, {0}));
AssertThat(update, IsTrue());
update = manager->resolve_lex_action(LexAction::Advance(2, {0}), LexAction::Error());
AssertThat(update, IsFalse());
});
describe("accept-token/advance conflicts", [&]() {
it("prefers the advance", [&]() {
update = manager->resolve_lex_action(LexAction::Accept(sym3, 3), LexAction::Advance(1, { 0 }));
AssertThat(update, IsTrue());
update = manager->resolve_lex_action(LexAction::Advance(1, { 0 }), LexAction::Accept(sym3, 3));
AssertThat(update, IsFalse());
});
});
describe("accept-token/accept-token conflicts", [&]() {
describe("when one token has a higher precedence than the other", [&]() {
it("prefers the token with the higher precedence", [&]() {
update = manager->resolve_lex_action(LexAction::Accept(sym3, 2), LexAction::Accept(sym2, 0));
AssertThat(update, IsFalse());
update = manager->resolve_lex_action(LexAction::Accept(sym2, 0), LexAction::Accept(sym3, 2));
AssertThat(update, IsTrue());
});
});
describe("when both tokens have the same precedence", [&]() {
it("prefers the token listed earlier in the grammar", [&]() {
update = manager->resolve_lex_action(LexAction::Accept(sym1, 0), LexAction::Accept(sym2, 0));
AssertThat(update, IsFalse());
update = manager->resolve_lex_action(LexAction::Accept(sym2, 0), LexAction::Accept(sym1, 0));
AssertThat(update, IsTrue());
});
});
});
before_each([&]() {
manager = new LexConflictManager(lex_grammar);
});
describe("syntactic conflicts", [&]() {
Symbol sym1(0);
Symbol sym2(1);
ParseConflictManager *manager;
before_each([&]() {
manager = new ParseConflictManager(parse_grammar, lex_grammar);
});
after_each([&]() {
delete manager;
});
it("favors non-errors over parse errors", [&]() {
update = manager->resolve_parse_action(sym1, ParseAction::Error(), ParseAction::Shift(2, { 0 }));
AssertThat(update, IsTrue());
update = manager->resolve_parse_action(sym1, ParseAction::Shift(2, { 0 }), ParseAction::Error());
AssertThat(update, IsFalse());
});
describe("shift/reduce conflicts", [&]() {
describe("when the shift has higher precedence", [&]() {
ParseAction shift = ParseAction::Shift(2, { 3 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 1);
it("does not record a conflict", [&]() {
manager->resolve_parse_action(sym1, shift, reduce);
manager->resolve_parse_action(sym1, reduce, shift);
AssertThat(manager->conflicts(), IsEmpty());
});
it("favors the shift", [&]() {
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsTrue());
});
});
describe("when the reduce has higher precedence", [&]() {
ParseAction shift = ParseAction::Shift(2, { 1 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 3);
it("does not record a conflict", [&]() {
manager->resolve_parse_action(sym1, reduce, shift);
manager->resolve_parse_action(sym1, shift, reduce);
AssertThat(manager->conflicts(), IsEmpty());
});
it("favors the reduce", [&]() {
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsTrue());
});
});
describe("when the precedences are equal", [&]() {
ParseAction shift = ParseAction::Shift(2, { 0 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 0);
it("records a conflict", [&]() {
manager->resolve_parse_action(sym1, reduce, shift);
manager->resolve_parse_action(sym1, shift, reduce);
AssertThat(manager->conflicts(), Equals(vector<Conflict>({
Conflict("rule1: shift (precedence 0) / reduce rule2 (precedence 0)")
})));
});
it("favors the shift", [&]() {
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsTrue());
});
describe("when the symbols is a built-in symbol", [&]() {
it("records a conflict", [&]() {
manager->resolve_parse_action(rules::ERROR(), reduce, shift);
AssertThat(manager->conflicts()[0], Equals(
Conflict("ERROR: shift (precedence 0) / reduce rule2 (precedence 0)")
));
manager->resolve_parse_action(rules::END_OF_INPUT(), reduce, shift);
AssertThat(manager->conflicts()[1], Equals(
Conflict("END_OF_INPUT: shift (precedence 0) / reduce rule2 (precedence 0)")
));
});
});
});
describe("when the shift has conflicting precedences compared to the reduce", [&]() {
ParseAction shift = ParseAction::Shift(2, { 0, 1, 3 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 2);
it("records a conflict", [&]() {
manager->resolve_parse_action(sym1, reduce, shift);
manager->resolve_parse_action(sym1, shift, reduce);
AssertThat(manager->conflicts(), Equals(vector<Conflict>({
Conflict("rule1: shift (precedence 0, 1, 3) / reduce rule2 (precedence 2)")
})));
});
it("favors the shift", [&]() {
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsTrue());
});
});
});
describe("reduce/reduce conflicts", [&]() {
describe("when one action has higher precedence", [&]() {
ParseAction left = ParseAction::Reduce(sym2, 1, 0);
ParseAction right = ParseAction::Reduce(sym2, 1, 3);
it("favors that action", [&]() {
AssertThat(manager->resolve_parse_action(sym1, left, right), IsTrue());
AssertThat(manager->resolve_parse_action(sym1, right, left), IsFalse());
});
it("does not record a conflict", [&]() {
manager->resolve_parse_action(sym1, left, right);
manager->resolve_parse_action(sym1, right, left);
AssertThat(manager->conflicts(), IsEmpty());
});
});
describe("when the actions have the same precedence", [&]() {
ParseAction left = ParseAction::Reduce(sym1, 1, 0);
ParseAction right = ParseAction::Reduce(sym2, 1, 0);
it("favors the symbol listed earlier in the grammar", [&]() {
AssertThat(manager->resolve_parse_action(sym1, right, left), IsTrue());
AssertThat(manager->resolve_parse_action(sym1, left, right), IsFalse());
});
it("records a conflict", [&]() {
manager->resolve_parse_action(sym1, left, right);
manager->resolve_parse_action(sym1, right, left);
AssertThat(manager->conflicts(), Equals(vector<Conflict>({
Conflict("rule1: reduce rule2 (precedence 0) / reduce rule1 (precedence 0)"),
Conflict("rule1: reduce rule1 (precedence 0) / reduce rule2 (precedence 0)")
})));
});
});
});
after_each([&]() {
delete manager;
});
it("favors non-errors over lexical errors", [&]() {
update = manager->resolve_lex_action(LexAction::Error(), LexAction::Advance(2, {0}));
AssertThat(update, IsTrue());
update = manager->resolve_lex_action(LexAction::Advance(2, {0}), LexAction::Error());
AssertThat(update, IsFalse());
});
describe("accept-token/advance conflicts", [&]() {
it("prefers the advance", [&]() {
update = manager->resolve_lex_action(LexAction::Accept(sym3, 3), LexAction::Advance(1, { 0 }));
AssertThat(update, IsTrue());
update = manager->resolve_lex_action(LexAction::Advance(1, { 0 }), LexAction::Accept(sym3, 3));
AssertThat(update, IsFalse());
});
});
describe("accept-token/accept-token conflicts", [&]() {
describe("when one token has a higher precedence than the other", [&]() {
it("prefers the token with the higher precedence", [&]() {
update = manager->resolve_lex_action(LexAction::Accept(sym3, 2), LexAction::Accept(sym2, 0));
AssertThat(update, IsFalse());
update = manager->resolve_lex_action(LexAction::Accept(sym2, 0), LexAction::Accept(sym3, 2));
AssertThat(update, IsTrue());
});
});
describe("when both tokens have the same precedence", [&]() {
it("prefers the token listed earlier in the grammar", [&]() {
update = manager->resolve_lex_action(LexAction::Accept(sym1, 0), LexAction::Accept(sym2, 0));
AssertThat(update, IsFalse());
update = manager->resolve_lex_action(LexAction::Accept(sym2, 0), LexAction::Accept(sym1, 0));
AssertThat(update, IsTrue());
});
});
});
});
describe("syntactic conflicts", [&]() {
Symbol sym1(0);
Symbol sym2(1);
ParseConflictManager *manager;
before_each([&]() {
manager = new ParseConflictManager(parse_grammar, lex_grammar);
});
after_each([&]() {
delete manager;
});
it("favors non-errors over parse errors", [&]() {
update = manager->resolve_parse_action(sym1, ParseAction::Error(), ParseAction::Shift(2, { 0 }));
AssertThat(update, IsTrue());
update = manager->resolve_parse_action(sym1, ParseAction::Shift(2, { 0 }), ParseAction::Error());
AssertThat(update, IsFalse());
});
describe("shift/reduce conflicts", [&]() {
describe("when the shift has higher precedence", [&]() {
ParseAction shift = ParseAction::Shift(2, { 3 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 1);
it("does not record a conflict", [&]() {
manager->resolve_parse_action(sym1, shift, reduce);
manager->resolve_parse_action(sym1, reduce, shift);
AssertThat(manager->conflicts(), IsEmpty());
});
it("favors the shift", [&]() {
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsTrue());
});
});
describe("when the reduce has higher precedence", [&]() {
ParseAction shift = ParseAction::Shift(2, { 1 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 3);
it("does not record a conflict", [&]() {
manager->resolve_parse_action(sym1, reduce, shift);
manager->resolve_parse_action(sym1, shift, reduce);
AssertThat(manager->conflicts(), IsEmpty());
});
it("favors the reduce", [&]() {
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsTrue());
});
});
describe("when the precedences are equal", [&]() {
ParseAction shift = ParseAction::Shift(2, { 0 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 0);
it("records a conflict", [&]() {
manager->resolve_parse_action(sym1, reduce, shift);
manager->resolve_parse_action(sym1, shift, reduce);
AssertThat(manager->conflicts(), Equals(vector<Conflict>({
Conflict("rule1: shift (precedence 0) / reduce rule2 (precedence 0)")
})));
});
it("favors the shift", [&]() {
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsTrue());
});
describe("when the symbols is a built-in symbol", [&]() {
it("records a conflict", [&]() {
manager->resolve_parse_action(rules::ERROR(), reduce, shift);
AssertThat(manager->conflicts()[0], Equals(
Conflict("ERROR: shift (precedence 0) / reduce rule2 (precedence 0)")
));
manager->resolve_parse_action(rules::END_OF_INPUT(), reduce, shift);
AssertThat(manager->conflicts()[1], Equals(
Conflict("END_OF_INPUT: shift (precedence 0) / reduce rule2 (precedence 0)")
));
});
});
});
describe("when the shift has conflicting precedences compared to the reduce", [&]() {
ParseAction shift = ParseAction::Shift(2, { 0, 1, 3 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 2);
it("records a conflict", [&]() {
manager->resolve_parse_action(sym1, reduce, shift);
manager->resolve_parse_action(sym1, shift, reduce);
AssertThat(manager->conflicts(), Equals(vector<Conflict>({
Conflict("rule1: shift (precedence 0, 1, 3) / reduce rule2 (precedence 2)")
})));
});
it("favors the shift", [&]() {
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsTrue());
});
});
});
describe("reduce/reduce conflicts", [&]() {
describe("when one action has higher precedence", [&]() {
ParseAction left = ParseAction::Reduce(sym2, 1, 0);
ParseAction right = ParseAction::Reduce(sym2, 1, 3);
it("favors that action", [&]() {
AssertThat(manager->resolve_parse_action(sym1, left, right), IsTrue());
AssertThat(manager->resolve_parse_action(sym1, right, left), IsFalse());
});
it("does not record a conflict", [&]() {
manager->resolve_parse_action(sym1, left, right);
manager->resolve_parse_action(sym1, right, left);
AssertThat(manager->conflicts(), IsEmpty());
});
});
describe("when the actions have the same precedence", [&]() {
ParseAction left = ParseAction::Reduce(sym1, 1, 0);
ParseAction right = ParseAction::Reduce(sym2, 1, 0);
it("favors the symbol listed earlier in the grammar", [&]() {
AssertThat(manager->resolve_parse_action(sym1, right, left), IsTrue());
AssertThat(manager->resolve_parse_action(sym1, left, right), IsFalse());
});
it("records a conflict", [&]() {
manager->resolve_parse_action(sym1, left, right);
manager->resolve_parse_action(sym1, right, left);
AssertThat(manager->conflicts(), Equals(vector<Conflict>({
Conflict("rule1: reduce rule2 (precedence 0) / reduce rule1 (precedence 0)"),
Conflict("rule1: reduce rule1 (precedence 0) / reduce rule2 (precedence 0)")
})));
});
});
});
});
});
END_TEST

155
spec/compiler/build_tables/first_set_spec.cc
View file

@ -3,99 +3,98 @@
#include "compiler/build_tables/first_set.h"
#include "compiler/rules/metadata.h"
using std::set;
using namespace build_tables;
using namespace rules;
START_TEST
describe("computing FIRST sets", []() {
const SyntaxGrammar null_grammar;
const SyntaxGrammar null_grammar;
describe("for a sequence AB", [&]() {
it("ignores B when A cannot be blank", [&]() {
auto rule = seq({ i_token(0), i_token(1) });
describe("for a sequence AB", [&]() {
it("ignores B when A cannot be blank", [&]() {
auto rule = seq({ i_token(0), i_token(1) });
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
})));
});
it("includes FIRST(B) when A can be blank", [&]() {
auto rule = seq({
choice({
i_token(0),
blank() }),
i_token(1) });
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(1, SymbolOptionToken)
})));
});
it("includes FIRST(A's right hand side) when A is a non-terminal", [&]() {
auto rule = choice({
seq({
i_token(0),
i_token(1) }),
i_sym(0) });
SyntaxGrammar grammar({
{ "rule0", seq({
i_token(2),
i_token(3),
i_token(4) }) }
}, {});
AssertThat(first_set(rule, grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(2, SymbolOptionToken),
})));
});
it("includes FIRST(B) when A is a non-terminal and its expansion can be blank", [&]() {
auto rule = seq({
i_sym(0),
i_token(1) });
SyntaxGrammar grammar({
{ "rule0", choice({
i_token(0),
blank() }) }
}, {});
AssertThat(first_set(rule, grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(1, SymbolOptionToken),
})));
});
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
})));
});
describe("when there are left-recursive rules", [&]() {
it("terminates", [&]() {
SyntaxGrammar grammar({
{ "rule0", choice({
seq({ i_sym(0), i_token(10) }),
i_token(11),
}) },
}, {});
it("includes FIRST(B) when A can be blank", [&]() {
auto rule = seq({
choice({
i_token(0),
blank() }),
i_token(1) });
auto rule = i_sym(0);
AssertThat(first_set(rule, grammar), Equals(set<Symbol>({
Symbol(11, SymbolOptionToken)
})));
});
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(1, SymbolOptionToken)
})));
});
it("ignores metadata rules", [&]() {
auto rule = make_shared<Metadata>(i_token(3), map<rules::MetadataKey, int>());
it("includes FIRST(A's right hand side) when A is a non-terminal", [&]() {
auto rule = choice({
seq({
i_token(0),
i_token(1) }),
i_sym(0) });
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol(3, SymbolOptionToken),
})));
SyntaxGrammar grammar({
{ "rule0", seq({
i_token(2),
i_token(3),
i_token(4) }) }
}, {});
AssertThat(first_set(rule, grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(2, SymbolOptionToken),
})));
});
it("includes FIRST(B) when A is a non-terminal and its expansion can be blank", [&]() {
auto rule = seq({
i_sym(0),
i_token(1) });
SyntaxGrammar grammar({
{ "rule0", choice({
i_token(0),
blank() }) }
}, {});
AssertThat(first_set(rule, grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(1, SymbolOptionToken),
})));
});
});
describe("when there are left-recursive rules", [&]() {
it("terminates", [&]() {
SyntaxGrammar grammar({
{ "rule0", choice({
seq({ i_sym(0), i_token(10) }),
i_token(11),
}) },
}, {});
auto rule = i_sym(0);
AssertThat(first_set(rule, grammar), Equals(set<Symbol>({
Symbol(11, SymbolOptionToken)
})));
});
});
it("ignores metadata rules", [&]() {
auto rule = make_shared<Metadata>(i_token(3), map<rules::MetadataKey, int>());
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol(3, SymbolOptionToken),
})));
});
});
END_TEST

68
spec/compiler/build_tables/get_metadata_spec.cc
View file

@ -8,44 +8,44 @@ using namespace build_tables;
START_TEST
describe("getting metadata for rules", []() {
MetadataKey key1 = MetadataKey(100);
MetadataKey key2 = MetadataKey(101);
rule_ptr rule;
MetadataKey key1 = MetadataKey(100);
MetadataKey key2 = MetadataKey(101);
rule_ptr rule;
describe("when given a metadata rule", [&]() {
before_each([&]() {
rule = make_shared<Metadata>(sym("x"), map<MetadataKey, int>({
{ key1, 1 },
{ key2, 2 },
}));
});
it("returns the value for the given key", [&]() {
AssertThat(get_metadata(rule, key1), Equals(1));
AssertThat(get_metadata(rule, key2), Equals(2));
});
it("returns 0 if the rule does not have the key", [&]() {
AssertThat(get_metadata(rule, MetadataKey(0)), Equals(0));
});
describe("when the rule contains another metadata rule", [&]() {
it("also gets metadata from the inner metadata rule", [&]() {
rule = make_shared<Metadata>(make_shared<Metadata>(sym("x"), map<MetadataKey, int>({
{ key1, 1 }
})), map<MetadataKey, int>());
AssertThat(get_metadata(rule, key1), Equals(1));
});
});
describe("when given a metadata rule", [&]() {
before_each([&]() {
rule = make_shared<Metadata>(sym("x"), map<MetadataKey, int>({
{ key1, 1 },
{ key2, 2 },
}));
});
describe("when given a non-metadata rule", [&]() {
it("returns 0", [&]() {
auto rule = sym("x");
AssertThat(get_metadata(rule, key1), Equals(0));
});
it("returns the value for the given key", [&]() {
AssertThat(get_metadata(rule, key1), Equals(1));
AssertThat(get_metadata(rule, key2), Equals(2));
});
it("returns 0 if the rule does not have the key", [&]() {
AssertThat(get_metadata(rule, MetadataKey(0)), Equals(0));
});
describe("when the rule contains another metadata rule", [&]() {
it("also gets metadata from the inner metadata rule", [&]() {
rule = make_shared<Metadata>(make_shared<Metadata>(sym("x"), map<MetadataKey, int>({
{ key1, 1 }
})), map<MetadataKey, int>());
AssertThat(get_metadata(rule, key1), Equals(1));
});
});
});
describe("when given a non-metadata rule", [&]() {
it("returns 0", [&]() {
auto rule = sym("x");
AssertThat(get_metadata(rule, key1), Equals(0));
});
});
});
END_TEST

34
spec/compiler/build_tables/item_set_closure_spec.cc
View file

@ -9,25 +9,25 @@ using namespace rules;
START_TEST
describe("computing closures of item sets", []() {
SyntaxGrammar grammar({
{ "E", seq({
i_sym(1),
i_token(11) }) },
{ "T", seq({
i_token(12),
i_token(13) }) },
}, {});
SyntaxGrammar grammar({
{ "E", seq({
i_sym(1),
i_token(11) }) },
{ "T", seq({
i_token(12),
i_token(13) }) },
}, {});
it("adds items at the beginnings of referenced rules", [&]() {
ParseItemSet item_set = item_set_closure(ParseItem(Symbol(0), grammar.rule(Symbol(0)), 0),
{ Symbol(10, SymbolOptionToken) },
grammar);
it("adds items at the beginnings of referenced rules", [&]() {
ParseItemSet item_set = item_set_closure(ParseItem(Symbol(0), grammar.rule(Symbol(0)), 0),
{ Symbol(10, SymbolOptionToken) },
grammar);
AssertThat(item_set, Equals(ParseItemSet({
{ ParseItem(Symbol(1), grammar.rule(Symbol(1)), 0), { Symbol(11, SymbolOptionToken) } },
{ ParseItem(Symbol(0), grammar.rule(Symbol(0)), 0), { Symbol(10, SymbolOptionToken) } },
})));
});
AssertThat(item_set, Equals(ParseItemSet({
{ ParseItem(Symbol(1), grammar.rule(Symbol(1)), 0), { Symbol(11, SymbolOptionToken) } },
{ ParseItem(Symbol(0), grammar.rule(Symbol(0)), 0), { Symbol(10, SymbolOptionToken) } },
})));
});
});
END_TEST

60
spec/compiler/build_tables/item_set_transitions_spec.cc
View file

@ -8,43 +8,43 @@ using namespace build_tables;
START_TEST
describe("lexical item set transitions", []() {
describe("when two items in the set have transitions on the same character", [&]() {
it("merges the transitions by computing the union of the two item sets", [&]() {
LexItemSet set1({
LexItem(Symbol(1), character({ {'a', 'f'} })),
LexItem(Symbol(2), character({ {'e', 'x'} })) });
describe("when two items in the set have transitions on the same character", [&]() {
it("merges the transitions by computing the union of the two item sets", [&]() {
LexItemSet set1({
LexItem(Symbol(1), character({ {'a', 'f'} })),
LexItem(Symbol(2), character({ {'e', 'x'} })) });
AssertThat(char_transitions(set1), Equals(map<CharacterSet, LexItemSet>({
{ CharacterSet({ {'a', 'd'} }), LexItemSet({
LexItem(Symbol(1), blank()) }) },
{ CharacterSet({ {'e', 'f'} }), LexItemSet({
LexItem(Symbol(1), blank()),
LexItem(Symbol(2), blank()) }) },
{ CharacterSet({ {'g', 'x'} }), LexItemSet({
LexItem(Symbol(2), blank()) }) },
})));
});
AssertThat(char_transitions(set1), Equals(map<CharacterSet, LexItemSet>({
{ CharacterSet({ {'a', 'd'} }), LexItemSet({
LexItem(Symbol(1), blank()) }) },
{ CharacterSet({ {'e', 'f'} }), LexItemSet({
LexItem(Symbol(1), blank()),
LexItem(Symbol(2), blank()) }) },
{ CharacterSet({ {'g', 'x'} }), LexItemSet({
LexItem(Symbol(2), blank()) }) },
})));
});
});
});
describe("syntactic item set transitions", [&]() {
SyntaxGrammar grammar({
{ "A", blank() },
{ "B", i_token(21) },
}, {}, set<Symbol>());
SyntaxGrammar grammar({
{ "A", blank() },
{ "B", i_token(21) },
}, {}, set<Symbol>());
it("computes the closure of the new item sets", [&]() {
ParseItemSet set1({
{ ParseItem(Symbol(0), seq({ i_token(22), i_sym(1) }), 3), { Symbol(23, SymbolOptionToken) } },
});
AssertThat(sym_transitions(set1, grammar), Equals(map<Symbol, ParseItemSet>({
{ Symbol(22, SymbolOptionToken), ParseItemSet({
{ ParseItem(Symbol(0), i_sym(1), 4), { Symbol(23, SymbolOptionToken) } },
{ ParseItem(Symbol(1), i_token(21), 0), { Symbol(23, SymbolOptionToken) } },
}) },
})));
it("computes the closure of the new item sets", [&]() {
ParseItemSet set1({
{ ParseItem(Symbol(0), seq({ i_token(22), i_sym(1) }), 3), { Symbol(23, SymbolOptionToken) } },
});
AssertThat(sym_transitions(set1, grammar), Equals(map<Symbol, ParseItemSet>({
{ Symbol(22, SymbolOptionToken), ParseItemSet({
{ ParseItem(Symbol(0), i_sym(1), 4), { Symbol(23, SymbolOptionToken) } },
{ ParseItem(Symbol(1), i_token(21), 0), { Symbol(23, SymbolOptionToken) } },
}) },
})));
});
});
END_TEST

56
spec/compiler/build_tables/lex_item_spec.cc
View file

@ -9,36 +9,36 @@ using namespace build_tables;
START_TEST
describe("lex items", []() {
describe("determining if an item is the start of a token", [&]() {
Symbol sym(1);
rule_ptr token_start = make_shared<Metadata>(str("a"), map<MetadataKey, int>({
{ START_TOKEN, 1 }
}));
describe("determining if an item is the start of a token", [&]() {
Symbol sym(1);
rule_ptr token_start = make_shared<Metadata>(str("a"), map<MetadataKey, int>({
{ START_TOKEN, 1 }
}));
it("returns true for rules designated as token starts", [&]() {
LexItem item(sym, token_start);
AssertThat(item.is_token_start(), IsTrue());
});
it("returns false for rules not designated as token starts", [&]() {
AssertThat(LexItem(sym, make_shared<Metadata>(str("a"), map<MetadataKey, int>({
{ START_TOKEN, 0 }
}))).is_token_start(), IsFalse());
AssertThat(LexItem(sym, str("a")).is_token_start(), IsFalse());
});
describe("when given a sequence containing a token start", [&]() {
it("returns true when the rule before the token start may be blank", [&]() {
LexItem item(sym, seq({ repeat(str("a")), token_start }));
AssertThat(item.is_token_start(), IsTrue());
});
it("returns false when the rule before the token start cannot be blank", [&]() {
LexItem item(sym, seq({ str("a"), token_start }));
AssertThat(item.is_token_start(), IsFalse());
});
});
it("returns true for rules designated as token starts", [&]() {
LexItem item(sym, token_start);
AssertThat(item.is_token_start(), IsTrue());
});
it("returns false for rules not designated as token starts", [&]() {
AssertThat(LexItem(sym, make_shared<Metadata>(str("a"), map<MetadataKey, int>({
{ START_TOKEN, 0 }
}))).is_token_start(), IsFalse());
AssertThat(LexItem(sym, str("a")).is_token_start(), IsFalse());
});
describe("when given a sequence containing a token start", [&]() {
it("returns true when the rule before the token start may be blank", [&]() {
LexItem item(sym, seq({ repeat(str("a")), token_start }));
AssertThat(item.is_token_start(), IsTrue());
});
it("returns false when the rule before the token start cannot be blank", [&]() {
LexItem item(sym, seq({ str("a"), token_start }));
AssertThat(item.is_token_start(), IsFalse());
});
});
});
});
END_TEST

106
spec/compiler/build_tables/merge_transitions_spec.cc
View file

@ -7,69 +7,69 @@ using namespace build_tables;
START_TEST
describe("merging character set transitions", []() {
typedef map<CharacterSet, int> int_map;
typedef map<CharacterSet, int> int_map;
auto do_merge = [&](int_map *left, const pair<CharacterSet, int> &new_pair) {
merge_char_transition<int>(left, new_pair, [](int *l, const int *r) {
*l = *l | *r;
});
};
describe("when none of the transitions intersect", [&]() {
it("returns the union of the two sets of transitions", [&]() {
int_map map({
{ CharacterSet({ 'a', 'c' }), 1 },
{ CharacterSet({ 'x', 'y' }), 2 },
{ CharacterSet({ '1', '9' }), 4 },
});
do_merge(&map, { CharacterSet({ ' ' }), 8 });
do_merge(&map, { CharacterSet({ '\t' }), 16 });
AssertThat(map, Equals(int_map({
{ CharacterSet({ 'a', 'c' }), 1 },
{ CharacterSet({ 'x', 'y' }), 2 },
{ CharacterSet({ '1', '9' }), 4 },
{ CharacterSet({ ' ' }), 8 },
{ CharacterSet({ '\t' }), 16 },
})));
});
auto do_merge = [&](int_map *left, const pair<CharacterSet, int> &new_pair) {
merge_char_transition<int>(left, new_pair, [](int *l, const int *r) {
*l = *l | *r;
});
};
describe("when transitions intersect", [&]() {
it("merges the intersecting transitions using the provided function", [&]() {
int_map map({
{ CharacterSet({ {'a', 'f'}, {'A', 'F'} }), 1 },
{ CharacterSet({ {'0', '9'} }), 2 },
});
describe("when none of the transitions intersect", [&]() {
it("returns the union of the two sets of transitions", [&]() {
int_map map({
{ CharacterSet({ 'a', 'c' }), 1 },
{ CharacterSet({ 'x', 'y' }), 2 },
{ CharacterSet({ '1', '9' }), 4 },
});
do_merge(&map, { CharacterSet({ 'c' }), 4 });
do_merge(&map, { CharacterSet({ '3' }), 8 });
do_merge(&map, { CharacterSet({ ' ' }), 8 });
do_merge(&map, { CharacterSet({ '\t' }), 16 });
AssertThat(map, Equals(int_map({
{ CharacterSet({ {'a', 'b'}, {'d', 'f'}, {'A', 'F'} }), 1 },
{ CharacterSet({ {'c'} }), 5 },
{ CharacterSet({ {'0', '2'}, {'4', '9'} }), 2 },
{ CharacterSet({ '3' }), 10 },
})));
});
AssertThat(map, Equals(int_map({
{ CharacterSet({ 'a', 'c' }), 1 },
{ CharacterSet({ 'x', 'y' }), 2 },
{ CharacterSet({ '1', '9' }), 4 },
{ CharacterSet({ ' ' }), 8 },
{ CharacterSet({ '\t' }), 16 },
})));
});
});
describe("when two of the right transitions intersect the same left transition", [&]() {
it("splits the left-hand transition correctly", [&]() {
int_map map1({
{ CharacterSet({ 'a', 'c' }), 1 },
});
describe("when transitions intersect", [&]() {
it("merges the intersecting transitions using the provided function", [&]() {
int_map map({
{ CharacterSet({ {'a', 'f'}, {'A', 'F'} }), 1 },
{ CharacterSet({ {'0', '9'} }), 2 },
});
do_merge(&map1, { CharacterSet({ 'a' }), 2 });
do_merge(&map1, { CharacterSet({ 'c' }), 4 });
do_merge(&map, { CharacterSet({ 'c' }), 4 });
do_merge(&map, { CharacterSet({ '3' }), 8 });
AssertThat(map1, Equals(int_map({
{ CharacterSet({ 'a' }), 3 },
{ CharacterSet({ 'c' }), 5 },
})));
});
AssertThat(map, Equals(int_map({
{ CharacterSet({ {'a', 'b'}, {'d', 'f'}, {'A', 'F'} }), 1 },
{ CharacterSet({ {'c'} }), 5 },
{ CharacterSet({ {'0', '2'}, {'4', '9'} }), 2 },
{ CharacterSet({ '3' }), 10 },
})));
});
});
describe("when two of the right transitions intersect the same left transition", [&]() {
it("splits the left-hand transition correctly", [&]() {
int_map map1({
{ CharacterSet({ 'a', 'c' }), 1 },
});
do_merge(&map1, { CharacterSet({ 'a' }), 2 });
do_merge(&map1, { CharacterSet({ 'c' }), 4 });
AssertThat(map1, Equals(int_map({
{ CharacterSet({ 'a' }), 3 },
{ CharacterSet({ 'c' }), 5 },
})));
});
});
});
END_TEST

124
spec/compiler/build_tables/rule_can_be_blank_spec.cc
View file

@ -9,72 +9,72 @@ using build_tables::rule_can_be_blank;
START_TEST
describe("checking if rules can be blank", [&]() {
rule_ptr rule;
rule_ptr rule;
it("returns false for basic rules", [&]() {
AssertThat(rule_can_be_blank(i_sym(3)), IsFalse());
AssertThat(rule_can_be_blank(str("x")), IsFalse());
AssertThat(rule_can_be_blank(pattern("x")), IsFalse());
it("returns false for basic rules", [&]() {
AssertThat(rule_can_be_blank(i_sym(3)), IsFalse());
AssertThat(rule_can_be_blank(str("x")), IsFalse());
AssertThat(rule_can_be_blank(pattern("x")), IsFalse());
});
it("returns true for blanks", [&]() {
AssertThat(rule_can_be_blank(blank()), IsTrue());
});
it("returns true for repeats", [&]() {
AssertThat(rule_can_be_blank(repeat(str("x"))), IsTrue());
});
it("returns true for choices iff one or more sides can be blank", [&]() {
rule = choice({ sym("x"), blank() });
AssertThat(rule_can_be_blank(rule), IsTrue());
rule = choice({ blank(), sym("x") });
AssertThat(rule_can_be_blank(rule), IsTrue());
rule = choice({ sym("x"), sym("y") });
AssertThat(rule_can_be_blank(rule), IsFalse());
});
it("returns true for sequences iff both sides can be blank", [&]() {
rule = seq({ blank(), str("x") });
AssertThat(rule_can_be_blank(rule), IsFalse());
rule = seq({ str("x"), blank() });
AssertThat(rule_can_be_blank(rule), IsFalse());
rule = seq({ blank(), choice({ sym("x"), blank() }) });
AssertThat(rule_can_be_blank(rule), IsTrue());
});
it("ignores metadata rules", [&]() {
rule = make_shared<rules::Metadata>(blank(), map<rules::MetadataKey, int>());
AssertThat(rule_can_be_blank(rule), IsTrue());
rule = make_shared<rules::Metadata>(sym("one"), map<rules::MetadataKey, int>());
AssertThat(rule_can_be_blank(rule), IsFalse());
});
describe("checking recursively (by expanding non-terminals)", [&]() {
SyntaxGrammar grammar({
{ "A", choice({
seq({ i_sym(0), i_token(11) }),
blank() }) },
{ "B", choice({
seq({ i_sym(1), i_token(12) }),
i_token(13) }) },
}, {}, set<Symbol>());
it("terminates for left-recursive rules that can be blank", [&]() {
rule = i_sym(0);
AssertThat(rule_can_be_blank(rule, grammar), IsTrue());
});
it("returns true for blanks", [&]() {
AssertThat(rule_can_be_blank(blank()), IsTrue());
});
it("returns true for repeats", [&]() {
AssertThat(rule_can_be_blank(repeat(str("x"))), IsTrue());
});
it("returns true for choices iff one or more sides can be blank", [&]() {
rule = choice({ sym("x"), blank() });
AssertThat(rule_can_be_blank(rule), IsTrue());
rule = choice({ blank(), sym("x") });
AssertThat(rule_can_be_blank(rule), IsTrue());
rule = choice({ sym("x"), sym("y") });
AssertThat(rule_can_be_blank(rule), IsFalse());
});
it("returns true for sequences iff both sides can be blank", [&]() {
rule = seq({ blank(), str("x") });
AssertThat(rule_can_be_blank(rule), IsFalse());
rule = seq({ str("x"), blank() });
AssertThat(rule_can_be_blank(rule), IsFalse());
rule = seq({ blank(), choice({ sym("x"), blank() }) });
AssertThat(rule_can_be_blank(rule), IsTrue());
});
it("ignores metadata rules", [&]() {
rule = make_shared<rules::Metadata>(blank(), map<rules::MetadataKey, int>());
AssertThat(rule_can_be_blank(rule), IsTrue());
rule = make_shared<rules::Metadata>(sym("one"), map<rules::MetadataKey, int>());
AssertThat(rule_can_be_blank(rule), IsFalse());
});
describe("checking recursively (by expanding non-terminals)", [&]() {
SyntaxGrammar grammar({
{ "A", choice({
seq({ i_sym(0), i_token(11) }),
blank() }) },
{ "B", choice({
seq({ i_sym(1), i_token(12) }),
i_token(13) }) },
}, {}, set<Symbol>());
it("terminates for left-recursive rules that can be blank", [&]() {
rule = i_sym(0);
AssertThat(rule_can_be_blank(rule, grammar), IsTrue());
});
it("terminates for left-recursive rules that can't be blank", [&]() {
rule = i_sym(1);
AssertThat(rule_can_be_blank(rule, grammar), IsFalse());
});
it("terminates for left-recursive rules that can't be blank", [&]() {
rule = i_sym(1);
AssertThat(rule_can_be_blank(rule, grammar), IsFalse());
});
});
});
END_TEST

329
spec/compiler/build_tables/rule_transitions_spec.cc
View file

@ -9,181 +9,180 @@ using namespace build_tables;
START_TEST
describe("rule transitions", []() {
it("handles symbols", [&]() {
AssertThat(
sym_transitions(i_sym(1)),
Equals(rule_map<Symbol>({
{ Symbol(1), blank() }
})));
});
it("handles symbols", [&]() {
AssertThat(
sym_transitions(i_sym(1)),
Equals(rule_map<Symbol>({
{ Symbol(1), blank() }
})));
});
it("handles choices", [&]() {
AssertThat(
sym_transitions(choice({ i_sym(1), i_sym(2) })),
Equals(rule_map<Symbol>({
{ Symbol(1), blank() },
{ Symbol(2), blank() }
})));
});
it("handles choices", [&]() {
AssertThat(
sym_transitions(choice({ i_sym(1), i_sym(2) })),
Equals(rule_map<Symbol>({
{ Symbol(1), blank() },
{ Symbol(2), blank() }
})));
});
it("handles sequences", [&]() {
AssertThat(
sym_transitions(seq({ i_sym(1), i_sym(2) })),
Equals(rule_map<Symbol>({
{ Symbol(1), i_sym(2) }
})));
});
it("handles sequences", [&]() {
AssertThat(
sym_transitions(seq({ i_sym(1), i_sym(2) })),
Equals(rule_map<Symbol>({
{ Symbol(1), i_sym(2) }
})));
});
it("handles long sequences", [&]() {
AssertThat(
sym_transitions(seq({
it("handles long sequences", [&]() {
AssertThat(
sym_transitions(seq({
i_sym(1),
i_sym(2),
i_sym(3),
i_sym(4)
})),
Equals(rule_map<Symbol>({
{ Symbol(1), seq({ i_sym(2), i_sym(3), i_sym(4) }) }
})));
});
it("handles sequences whose left sides can be blank", [&]() {
AssertThat(
sym_transitions(seq({
choice({
i_sym(1),
i_sym(2),
i_sym(3),
i_sym(4)
})),
Equals(rule_map<Symbol>({
{ Symbol(1), seq({ i_sym(2), i_sym(3), i_sym(4) }) }
})));
});
blank(),
}),
seq({
i_sym(1),
i_sym(2)
})
})), Equals(rule_map<Symbol>({
{ Symbol(1), choice({ seq({ i_sym(1), i_sym(2) }), i_sym(2), }) }
})));
});
it("handles sequences whose left sides can be blank", [&]() {
AssertThat(
sym_transitions(seq({
choice({
i_sym(1),
blank(),
}),
it("handles choices with common starting symbols", [&]() {
AssertThat(
sym_transitions(
choice({
seq({ i_sym(1), i_sym(2) }),
seq({ i_sym(1), i_sym(3) }) })),
Equals(rule_map<Symbol>({
{ Symbol(1), choice({ i_sym(2), i_sym(3) }) }
})));
});
it("handles characters", [&]() {
AssertThat(
char_transitions(character({ '1' })),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ '1' }), blank() }
})));
});
it("handles choices between overlapping character sets", [&]() {
AssertThat(
char_transitions(choice({
seq({
character({ {'a', 's'} }),
sym("x") }),
seq({
character({ { 'm', 'z' } }),
sym("y") }) })),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ {'a','l'} }), sym("x") },
{ CharacterSet({ {'m','s'} }), choice({ sym("x"), sym("y") }) },
{ CharacterSet({ {'t','z'} }), sym("y") },
})));
});
it("handles choices between a subset and a superset of characters", [&]() {
AssertThat(
char_transitions(choice({
seq({
character({ {'a', 'c'} }),
sym("x") }),
seq({
character({ { 'a', 'z' } }),
sym("y") }) })),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ {'a', 'c'} }), choice({ sym("x"), sym("y") }) },
{ CharacterSet({ {'d', 'z'} }), sym("y") },
})));
AssertThat(
char_transitions(choice({
seq({
character({ { 'a', 'z' } }),
sym("x") }),
seq({
character({ {'a', 'c'} }),
sym("y") }) })),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ {'a', 'c'} }), choice({ sym("x"), sym("y") }) },
{ CharacterSet({ {'d', 'z'} }), sym("x") },
})));
});
it("handles blanks", [&]() {
AssertThat(char_transitions(blank()), Equals(rule_map<CharacterSet>({})));
});
it("handles repeats", [&]() {
rule_ptr rule = repeat(seq({ character({ 'a' }), character({ 'b' }) }));
AssertThat(
char_transitions(rule),
Equals(rule_map<CharacterSet>({
{
CharacterSet({ 'a' }),
seq({
i_sym(1),
i_sym(2)
character({ 'b' }),
rule,
})
})), Equals(rule_map<Symbol>({
{ Symbol(1), choice({ seq({ i_sym(1), i_sym(2) }), i_sym(2), }) }
})));
}})));
rule = repeat(character({ 'a' }));
AssertThat(
char_transitions(rule),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ 'a' }), rule }
})));
});
it("preserves metadata", [&]() {
map<MetadataKey, int> metadata_value({
{ PRECEDENCE, 5 }
});
it("handles choices with common starting symbols", [&]() {
AssertThat(
sym_transitions(
choice({
seq({ i_sym(1), i_sym(2) }),
seq({ i_sym(1), i_sym(3) }) })),
Equals(rule_map<Symbol>({
{ Symbol(1), choice({ i_sym(2), i_sym(3) }) }
})));
});
it("handles characters", [&]() {
AssertThat(
char_transitions(character({ '1' })),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ '1' }), blank() }
})));
});
it("handles choices between overlapping character sets", [&]() {
AssertThat(
char_transitions(choice({
seq({
character({ {'a', 's'} }),
sym("x") }),
seq({
character({ { 'm', 'z' } }),
sym("y") }) })),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ {'a','l'} }), sym("x") },
{ CharacterSet({ {'m','s'} }), choice({ sym("x"), sym("y") }) },
{ CharacterSet({ {'t','z'} }), sym("y") },
})));
});
it("handles choices between a subset and a superset of characters", [&]() {
AssertThat(
char_transitions(choice({
seq({
character({ {'a', 'c'} }),
sym("x") }),
seq({
character({ { 'a', 'z' } }),
sym("y") }) })),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ {'a', 'c'} }), choice({ sym("x"), sym("y") }) },
{ CharacterSet({ {'d', 'z'} }), sym("y") },
})));
AssertThat(
char_transitions(choice({
seq({
character({ { 'a', 'z' } }),
sym("x") }),
seq({
character({ {'a', 'c'} }),
sym("y") }) })),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ {'a', 'c'} }), choice({ sym("x"), sym("y") }) },
{ CharacterSet({ {'d', 'z'} }), sym("x") },
})));
});
it("handles blanks", [&]() {
AssertThat(char_transitions(blank()), Equals(rule_map<CharacterSet>({})));
});
it("handles repeats", [&]() {
rule_ptr rule = repeat(seq({ character({ 'a' }), character({ 'b' }) }));
AssertThat(
char_transitions(rule),
Equals(rule_map<CharacterSet>({
{
CharacterSet({ 'a' }),
seq({
character({ 'b' }),
rule,
})
}})));
rule = repeat(character({ 'a' }));
AssertThat(
char_transitions(rule),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ 'a' }), rule }
})));
});
it("preserves metadata", [&]() {
map<MetadataKey, int> metadata_value({
{ PRECEDENCE, 5 }
});
rule_ptr rule = make_shared<Metadata>(seq({ i_sym(1), i_sym(2) }), metadata_value);
AssertThat(
sym_transitions(rule),
Equals(rule_map<Symbol>({
{ Symbol(1), make_shared<Metadata>(i_sym(2), metadata_value)},
})));
});
describe("regression tests (somewhat redundant, should maybe be deleted later)", []() {
it("handles sequences that start with repeating characters", [&]() {
auto rule = seq({
choice({
repeat(character({ '"' }, false)),
blank(),
}),
character({ '"' }),
});
AssertThat(char_transitions(rule), Equals(rule_map<CharacterSet>({
{ CharacterSet({ '"' }).complement(), seq({
repeat(character({ '"' }, false)),
character({ '"' }), }) },
{ CharacterSet({ '"' }), blank() },
})));
});
rule_ptr rule = make_shared<Metadata>(seq({ i_sym(1), i_sym(2) }), metadata_value);
AssertThat(
sym_transitions(rule),
Equals(rule_map<Symbol>({
{ Symbol(1), make_shared<Metadata>(i_sym(2), metadata_value)},
})));
});
describe("regression tests (somewhat redundant, should maybe be deleted later)", []() {
it("handles sequences that start with repeating characters", [&]() {
auto rule = seq({
choice({
repeat(character({ '"' }, false)),
blank(),
}),
character({ '"' }),
});
AssertThat(char_transitions(rule), Equals(rule_map<CharacterSet>({
{ CharacterSet({ '"' }).complement(), seq({
repeat(character({ '"' }, false)),
character({ '"' }), }) },
{ CharacterSet({ '"' }), blank() },
})));
});
});
});
END_TEST

53
spec/compiler/compile_examples.cc
View file

@ -2,44 +2,45 @@
#include <fstream>
static string src_dir() {
const char * dir = getenv("TREESITTER_DIR");
if (!dir) dir = getenv("PWD");
return dir;
const char * dir = getenv("TREESITTER_DIR");
if (!dir) dir = getenv("PWD");
return dir;
}
namespace tree_sitter_examples {
extern const Grammar arithmetic;
extern const Grammar javascript;
extern const Grammar json;
extern const Grammar golang;
}
extern const Grammar arithmetic;
extern const Grammar javascript;
extern const Grammar json;
extern const Grammar golang;
} // namespace tree_sitter_examples
START_TEST
describe("compiling the example grammars", []() {
string example_parser_dir = src_dir() + "/examples/parsers/";
string example_parser_dir = src_dir() + "/examples/parsers/";
auto compile_grammar = [&](const Grammar &grammar, string language) {
it(("compiles the " + language + " grammar").c_str(), [&]() {
ofstream file(example_parser_dir + language + ".c");
auto compile_grammar = [&](const Grammar &grammar, string language) {
it(("compiles the " + language + " grammar").c_str(), [&]() {
ofstream file(example_parser_dir + language + ".c");
auto result = compile(grammar, language);
string code = get<0>(result);
vector<Conflict> conflicts = get<1>(result);
const GrammarError *error = get<2>(result);
auto result = compile(grammar, language);
string code = get<0>(result);
vector<Conflict> conflicts = get<1>(result);
const GrammarError *error = get<2>(result);
AssertThat(error, Equals((GrammarError *)nullptr));
// cout << "\n\nconflicts for " << language << ":\n" << get<1>(result);
AssertThat(error, Equals((GrammarError *)nullptr));
file << get<0>(result);
file.close();
});
};
file << get<0>(result);
file.close();
});
};
compile_grammar(tree_sitter_examples::arithmetic, "arithmetic");
compile_grammar(tree_sitter_examples::json, "json");
compile_grammar(tree_sitter_examples::javascript, "javascript");
compile_grammar(tree_sitter_examples::golang, "golang");
compile_grammar(tree_sitter_examples::arithmetic, "arithmetic");
compile_grammar(tree_sitter_examples::json, "json");
compile_grammar(tree_sitter_examples::javascript, "javascript");
compile_grammar(tree_sitter_examples::golang, "golang");
});
END_TEST

2
spec/compiler/compiler_specs.cc
View file

@ -1,5 +1,5 @@
#include "compiler/compiler_spec_helper.h"
int main(int argc, char *argv[]) {
return bandit::run(argc, argv);
return bandit::run(argc, argv);
}

48
spec/compiler/helpers/containers.h
View file

@ -17,35 +17,35 @@ using tree_sitter::rules::rule_ptr;
template<typename K>
class rule_map : public map<K, rule_ptr> {
public:
bool operator==(const map<K, rule_ptr> &other) const {
if (this->size() != other.size()) return false;
for (const auto &pair : *this) {
auto other_pair = other.find(pair.first);
if (other_pair == other.end()) return false;
if (!pair.second->operator==(*other_pair->second)) return false;
}
return true;
}
public:
bool operator==(const map<K, rule_ptr> &other) const {
if (this->size() != other.size()) return false;
for (const auto &pair : *this) {
auto other_pair = other.find(pair.first);
if (other_pair == other.end()) return false;
if (!pair.second->operator==(*other_pair->second)) return false;
}
return true;
}
rule_map(const initializer_list<pair<const K, rule_ptr>> &list) : map<K, rule_ptr>(list) {}
rule_map(const initializer_list<pair<const K, rule_ptr>> &list) : map<K, rule_ptr>(list) {}
};
class rule_list : public vector<pair<string, rule_ptr>> {
public:
bool operator==(const vector<pair<string, rule_ptr>> &other) const {
if (this->size() != other.size()) return false;
for (size_t i = 0; i < this->size(); i++) {
auto pair = this->operator[](i);
auto other_pair = other[i];
if (!pair.second->operator==(*other_pair.second))
return false;
}
return true;
}
public:
bool operator==(const vector<pair<string, rule_ptr>> &other) const {
if (this->size() != other.size()) return false;
for (size_t i = 0; i < this->size(); i++) {
auto pair = this->operator[](i);
auto other_pair = other[i];
if (!pair.second->operator==(*other_pair.second))
return false;
}
return true;
}
rule_list(const initializer_list<pair<string, rule_ptr>> &list) :
vector<pair<string, rule_ptr>>(list) {}
rule_list(const initializer_list<pair<string, rule_ptr>> &list) :
vector<pair<string, rule_ptr>>(list) {}
};

51
spec/compiler/helpers/equals_pointer.h
View file

@ -1,38 +1,37 @@
#ifndef HELPERS_EQUALS_POINTER_H_
#define HELPERS_EQUALS_POINTER_H_
#include "bandit/bandit.h"
#include <string>
namespace snowhouse {
using namespace std;
using namespace std;
template<typename ExpectedType>
struct EqualsPointerConstraint : Expression<EqualsPointerConstraint<ExpectedType>>
{
EqualsPointerConstraint(const ExpectedType& expected) : expected(expected) {}
template<typename ExpectedType>
struct EqualsPointerConstraint : Expression<EqualsPointerConstraint<ExpectedType>> {
EqualsPointerConstraint(const ExpectedType& expected) : expected(expected) {}
template<typename ActualType>
bool operator()(const ActualType& actual) const {
return *expected == *actual;
}
ExpectedType expected;
};
template<typename ExpectedType>
struct Stringizer<EqualsPointerConstraint<ExpectedType>>
{
static string ToString(const EqualsPointerConstraint<ExpectedType>& constraint) {
ostringstream builder;
builder << "pointer to " << snowhouse::Stringize(constraint.expected);
return builder.str();
}
};
template<typename ExpectedType>
inline EqualsPointerConstraint<ExpectedType> EqualsPointer(const ExpectedType& expected) {
return EqualsPointerConstraint<ExpectedType>(expected);
template<typename ActualType>
bool operator()(const ActualType& actual) const {
return *expected == *actual;
}
ExpectedType expected;
};
template<typename ExpectedType>
struct Stringizer<EqualsPointerConstraint<ExpectedType>> {
static string ToString(const EqualsPointerConstraint<ExpectedType>& constraint) {
ostringstream builder;
builder << "pointer to " << snowhouse::Stringize(constraint.expected);
return builder.str();
}
};
template<typename ExpectedType>
inline EqualsPointerConstraint<ExpectedType> EqualsPointer(const ExpectedType& expected) {
return EqualsPointerConstraint<ExpectedType>(expected);
}
}
#endif // HELPERS_EQUALS_POINTER_H_

68
spec/compiler/helpers/rule_helpers.cc
View file

@ -3,40 +3,40 @@
#include "compiler/rules/symbol.h"
namespace tree_sitter {
using std::make_shared;
using std::set;
using std::map;
using std::make_shared;
using std::set;
using std::map;
namespace rules {
rule_ptr character(const set<CharacterRange> &ranges) {
return make_shared<CharacterSet>(ranges);
}
rule_ptr character(const set<CharacterRange> &ranges, bool sign) {
if (sign)
return character(ranges);
else
return CharacterSet(ranges).complement().copy();
}
rule_ptr i_sym(size_t index) {
return make_shared<rules::Symbol>(index);
}
rule_ptr i_aux_sym(size_t index) {
return make_shared<rules::Symbol>(index, SymbolOptionAuxiliary);
}
rule_ptr i_token(size_t index) {
return make_shared<rules::Symbol>(index, SymbolOptionToken);
}
rule_ptr i_aux_token(size_t index) {
return make_shared<rules::Symbol>(index, SymbolOption(SymbolOptionAuxiliary|SymbolOptionToken));
}
rule_ptr metadata(rule_ptr rule, map<MetadataKey, int> values) {
return make_shared<Metadata>(rule, values);
}
namespace rules {
rule_ptr character(const set<CharacterRange> &ranges) {
return make_shared<CharacterSet>(ranges);
}
rule_ptr character(const set<CharacterRange> &ranges, bool sign) {
if (sign)
return character(ranges);
else
return CharacterSet(ranges).complement().copy();
}
rule_ptr i_sym(size_t index) {
return make_shared<rules::Symbol>(index);
}
rule_ptr i_aux_sym(size_t index) {
return make_shared<rules::Symbol>(index, SymbolOptionAuxiliary);
}
rule_ptr i_token(size_t index) {
return make_shared<rules::Symbol>(index, SymbolOptionToken);
}
rule_ptr i_aux_token(size_t index) {
return make_shared<rules::Symbol>(index, SymbolOption(SymbolOptionAuxiliary|SymbolOptionToken));
}
rule_ptr metadata(rule_ptr rule, map<MetadataKey, int> values) {
return make_shared<Metadata>(rule, values);
}
}
}

18
spec/compiler/helpers/rule_helpers.h
View file

@ -6,15 +6,15 @@
#include "compiler/rules/metadata.h"
namespace tree_sitter {
namespace rules {
rule_ptr metadata(rule_ptr, std::map<MetadataKey, int>);
rule_ptr character(const std::set<CharacterRange> &ranges);
rule_ptr character(const std::set<CharacterRange> &ranges, bool sign);
rule_ptr i_sym(size_t index);
rule_ptr i_aux_sym(size_t index);
rule_ptr i_token(size_t index);
rule_ptr i_aux_token(size_t index);
}
namespace rules {
rule_ptr metadata(rule_ptr, std::map<MetadataKey, int>);
rule_ptr character(const std::set<CharacterRange> &ranges);
rule_ptr character(const std::set<CharacterRange> &ranges, bool sign);
rule_ptr i_sym(size_t index);
rule_ptr i_aux_sym(size_t index);
rule_ptr i_token(size_t index);
rule_ptr i_aux_token(size_t index);
}
}
#endif

137
spec/compiler/helpers/stream_methods.h
View file

@ -12,74 +12,75 @@ using std::cout;
namespace std {
template<typename T>
inline std::ostream& operator<<(std::ostream &stream, const std::vector<T> &vector) {
stream << std::string("#<vector: ");
bool started = false;
for (auto item : vector) {
if (started) stream << std::string(", ");
stream << item;
started = true;
}
return stream << ">";
}
template<typename T>
inline std::ostream& operator<<(std::ostream &stream, const std::set<T> &set) {
stream << std::string("#<set: ");
bool started = false;
for (auto item : set) {
if (started) stream << std::string(", ");
stream << item;
started = true;
}
return stream << ">";
}
template<typename T>
inline std::ostream& operator<<(std::ostream &stream, const std::unordered_set<T> &set) {
stream << std::string("#<set: ");
bool started = false;
for (auto item : set) {
if (started) stream << std::string(", ");
stream << item;
started = true;
}
return stream << ">";
}
template<typename TKey, typename TValue>
inline std::ostream& operator<<(std::ostream &stream, const std::map<TKey, TValue> &map) {
stream << std::string("#<map: ");
bool started = false;
for (auto pair : map) {
if (started) stream << std::string(", ");
stream << pair.first;
stream << std::string(" => ");
stream << pair.second;
started = true;
}
return stream << ">";
}
template<typename TKey, typename TValue>
inline std::ostream& operator<<(std::ostream &stream, const std::unordered_map<TKey, TValue> &map) {
stream << std::string("#<map: ");
bool started = false;
for (auto pair : map) {
if (started) stream << std::string(", ");
stream << pair.first;
stream << std::string(" => ");
stream << pair.second;
started = true;
}
return stream << ">";
}
template<typename T1, typename T2>
inline std::ostream& operator<<(std::ostream &stream, const std::pair<T1, T2> &pair) {
return stream << "{" << pair.first << ", " << pair.second << "}";
}
template<typename T>
inline std::ostream& operator<<(std::ostream &stream, const std::vector<T> &vector) {
stream << std::string("#<vector: ");
bool started = false;
for (auto item : vector) {
if (started) stream << std::string(", ");
stream << item;
started = true;
}
return stream << ">";
}
template<typename T>
inline std::ostream& operator<<(std::ostream &stream, const std::set<T> &set) {
stream << std::string("#<set: ");
bool started = false;
for (auto item : set) {
if (started) stream << std::string(", ");
stream << item;
started = true;
}
return stream << ">";
}
template<typename T>
inline std::ostream& operator<<(std::ostream &stream, const std::unordered_set<T> &set) {
stream << std::string("#<set: ");
bool started = false;
for (auto item : set) {
if (started) stream << std::string(", ");
stream << item;
started = true;
}
return stream << ">";
}
template<typename TKey, typename TValue>
inline std::ostream& operator<<(std::ostream &stream, const std::map<TKey, TValue> &map) {
stream << std::string("#<map: ");
bool started = false;
for (auto pair : map) {
if (started) stream << std::string(", ");
stream << pair.first;
stream << std::string(" => ");
stream << pair.second;
started = true;
}
return stream << ">";
}
template<typename TKey, typename TValue>
inline std::ostream& operator<<(std::ostream &stream, const std::unordered_map<TKey, TValue> &map) {
stream << std::string("#<map: ");
bool started = false;
for (auto pair : map) {
if (started) stream << std::string(", ");
stream << pair.first;
stream << std::string(" => ");
stream << pair.second;
started = true;
}
return stream << ">";
}
template<typename T1, typename T2>
inline std::ostream& operator<<(std::ostream &stream, const std::pair<T1, T2> &pair) {
return stream << "{" << pair.first << ", " << pair.second << "}";
}
} // namespace std
#endif

162
spec/compiler/prepare_grammar/expand_repeats_spec.cc
View file

@ -9,105 +9,105 @@ using namespace rules;
using prepare_grammar::expand_repeats;
describe("expanding repeat rules in a grammar", []() {
it("replaces repeat rules with pairs of recursive rules", [&]() {
SyntaxGrammar grammar({
{ "rule0", repeat(i_token(0)) },
}, {}, set<Symbol>());
it("replaces repeat rules with pairs of recursive rules", [&]() {
SyntaxGrammar grammar({
{ "rule0", repeat(i_token(0)) },
}, {}, set<Symbol>());
auto match = expand_repeats(grammar);
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", i_aux_sym(0) },
})));
AssertThat(match.rules, Equals(rule_list({
{ "rule0", i_aux_sym(0) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({ seq({ i_token(0), i_aux_sym(0) }), blank() }) },
})));
});
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({ seq({ i_token(0), i_aux_sym(0) }), blank() }) },
})));
});
it("replaces repeats inside of sequences", [&]() {
SyntaxGrammar grammar({
{ "rule0", seq({ i_token(10), repeat(i_token(11)) }) },
}, {}, set<Symbol>());
it("replaces repeats inside of sequences", [&]() {
SyntaxGrammar grammar({
{ "rule0", seq({ i_token(10), repeat(i_token(11)) }) },
}, {}, set<Symbol>());
auto match = expand_repeats(grammar);
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", seq({ i_token(10), i_aux_sym(0) }) },
})));
AssertThat(match.rules, Equals(rule_list({
{ "rule0", seq({ i_token(10), i_aux_sym(0) }) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({ i_token(11), i_aux_sym(0) }),
blank() }) },
})));
});
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({ i_token(11), i_aux_sym(0) }),
blank() }) },
})));
});
it("replaces repeats inside of choices", [&]() {
SyntaxGrammar grammar({
{ "rule0", choice({ i_token(10), repeat(i_token(11)) }) },
}, {}, set<Symbol>());
it("replaces repeats inside of choices", [&]() {
SyntaxGrammar grammar({
{ "rule0", choice({ i_token(10), repeat(i_token(11)) }) },
}, {}, set<Symbol>());
auto match = expand_repeats(grammar);
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", choice({ i_token(10), i_aux_sym(0) }) },
})));
AssertThat(match.rules, Equals(rule_list({
{ "rule0", choice({ i_token(10), i_aux_sym(0) }) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({ i_token(11), i_aux_sym(0) }),
blank() }) },
})));
});
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({ i_token(11), i_aux_sym(0) }),
blank() }) },
})));
});
it("can replace multiple repeats in the same rule", [&]() {
SyntaxGrammar grammar({
{ "rule0", seq({ repeat(i_token(10)), repeat(i_token(11)) }) },
}, {}, set<Symbol>());
it("can replace multiple repeats in the same rule", [&]() {
SyntaxGrammar grammar({
{ "rule0", seq({ repeat(i_token(10)), repeat(i_token(11)) }) },
}, {}, set<Symbol>());
auto match = expand_repeats(grammar);
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", seq({ i_aux_sym(0), i_aux_sym(1) }) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({
i_token(10),
i_aux_sym(0) }),
blank() }) },
{ "rule0_repeat1", choice({
seq({
i_token(11),
i_aux_sym(1) }),
blank() }) },
})));
});
AssertThat(match.rules, Equals(rule_list({
{ "rule0", seq({ i_aux_sym(0), i_aux_sym(1) }) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({
i_token(10),
i_aux_sym(0) }),
blank() }) },
{ "rule0_repeat1", choice({
seq({
i_token(11),
i_aux_sym(1) }),
blank() }) },
})));
});
it("can replace repeats in multiple rules", [&]() {
SyntaxGrammar grammar({
{ "rule0", repeat(i_token(10)) },
{ "rule1", repeat(i_token(11)) },
}, {}, set<Symbol>());
it("can replace repeats in multiple rules", [&]() {
SyntaxGrammar grammar({
{ "rule0", repeat(i_token(10)) },
{ "rule1", repeat(i_token(11)) },
}, {}, set<Symbol>());
auto match = expand_repeats(grammar);
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", i_aux_sym(0) },
{ "rule1", i_aux_sym(1) },
})));
AssertThat(match.rules, Equals(rule_list({
{ "rule0", i_aux_sym(0) },
{ "rule1", i_aux_sym(1) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({ i_token(10), i_aux_sym(0) }),
blank() }) },
{ "rule1_repeat0", choice({
seq({ i_token(11), i_aux_sym(1) }),
blank() }) },
})));
});
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({ i_token(10), i_aux_sym(0) }),
blank() }) },
{ "rule1_repeat0", choice({
seq({ i_token(11), i_aux_sym(1) }),
blank() }) },
})));
});
});
END_TEST

84
spec/compiler/prepare_grammar/expand_tokens_spec.cc
View file

@ -9,56 +9,56 @@ using namespace rules;
using prepare_grammar::expand_tokens;
describe("expanding token rules", []() {
it("replaces regex patterns with their expansion", [&]() {
LexicalGrammar grammar({
{ "rule_A", seq({
i_sym(10),
pattern("x*"),
i_sym(11) }) },
}, {});
it("replaces regex patterns with their expansion", [&]() {
LexicalGrammar grammar({
{ "rule_A", seq({
i_sym(10),
pattern("x*"),
i_sym(11) }) },
}, {});
auto result = expand_tokens(grammar);
auto result = expand_tokens(grammar);
AssertThat(result.second, Equals((const GrammarError *)nullptr));
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({
i_sym(10),
repeat(character({ 'x' })),
i_sym(11) }) },
})));
});
AssertThat(result.second, Equals((const GrammarError *)nullptr));
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({
i_sym(10),
repeat(character({ 'x' })),
i_sym(11) }) },
})));
});
it("replaces string rules with a sequence of characters", [&]() {
LexicalGrammar grammar({
{ "rule_A", seq({
i_sym(10),
str("xyz"),
i_sym(11) }) },
}, {});
it("replaces string rules with a sequence of characters", [&]() {
LexicalGrammar grammar({
{ "rule_A", seq({
i_sym(10),
str("xyz"),
i_sym(11) }) },
}, {});
auto result = expand_tokens(grammar);
auto result = expand_tokens(grammar);
AssertThat(result.second, Equals((const GrammarError *)nullptr));
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({
i_sym(10),
seq({ character({ 'x' }), character({ 'y' }), character({ 'z' }) }),
i_sym(11) }) },
})));
});
AssertThat(result.second, Equals((const GrammarError *)nullptr));
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({
i_sym(10),
seq({ character({ 'x' }), character({ 'y' }), character({ 'z' }) }),
i_sym(11) }) },
})));
});
it("returns an error when the grammar contains an invalid regex", [&]() {
LexicalGrammar grammar({
{ "rule_A", seq({
pattern("("),
str("xyz"),
pattern("[") }) },
}, {});
it("returns an error when the grammar contains an invalid regex", [&]() {
LexicalGrammar grammar({
{ "rule_A", seq({
pattern("("),
str("xyz"),
pattern("[") }) },
}, {});
auto result = expand_tokens(grammar);
auto result = expand_tokens(grammar);
AssertThat(result.second, EqualsPointer(new GrammarError(GrammarErrorTypeRegex, "unmatched open paren")));
});
AssertThat(result.second, EqualsPointer(new GrammarError(GrammarErrorTypeRegex, "unmatched open paren")));
});
});
END_TEST

294
spec/compiler/prepare_grammar/extract_tokens_spec.cc
View file

@ -12,174 +12,174 @@ using prepare_grammar::extract_tokens;
using prepare_grammar::InternedGrammar;
describe("extracting tokens from a grammar", []() {
it("moves string rules into the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", seq({ str("ab"), i_sym(0) }) }
},
set<Symbol>(),
set<char>()
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0) }) }
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, Equals(rule_list({
{ "'ab'", str("ab") },
})));
it("moves string rules into the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", seq({ str("ab"), i_sym(0) }) }
},
set<Symbol>(),
set<char>()
});
it("moves pattern rules into the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", seq({ pattern("a+"), i_sym(0) }) }
},
set<Symbol>(),
set<char>()
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0) }) }
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, Equals(rule_list({
{ "'ab'", str("ab") },
})));
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0) }) }
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, Equals(rule_list({
{ "/a+/", pattern("a+") },
})));
it("moves pattern rules into the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", seq({ pattern("a+"), i_sym(0) }) }
},
set<Symbol>(),
set<char>()
});
it("moves other rules marked as tokens into the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", seq({
token(seq({ pattern("."), choice({ str("a"), str("b") }) })),
i_sym(0) }) }
},
set<Symbol>(),
set<char>()
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0) }) }
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, Equals(rule_list({
{ "/a+/", pattern("a+") },
})));
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0) }) }
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, Equals(rule_list({
{ "(seq /./ (choice 'a' 'b'))", token(seq({ pattern("."), choice({ str("a"), str("b") }) })) },
})));
it("moves other rules marked as tokens into the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", seq({
token(seq({ pattern("."), choice({ str("a"), str("b") }) })),
i_sym(0) }) }
},
set<Symbol>(),
set<char>()
});
it("does not extract blanks", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", choice({ i_sym(0), blank() }) },
},
set<Symbol>(),
set<char>()
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0) }) }
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, Equals(rule_list({
{ "(seq /./ (choice 'a' 'b'))", token(seq({ pattern("."), choice({ str("a"), str("b") }) })) },
})));
});
AssertThat(result.first.rules, Equals(rule_list({
it("does not extract blanks", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", choice({ i_sym(0), blank() }) },
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, IsEmpty())
},
set<Symbol>(),
set<char>()
});
it("does not create duplicate tokens in the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", seq({ str("ab"), i_sym(0), str("ab") }) },
},
set<Symbol>(),
set<char>()
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", choice({ i_sym(0), blank() }) },
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, IsEmpty())
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0), i_aux_token(0) }) }
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, Equals(rule_list({
{ "'ab'", str("ab") },
})))
it("does not create duplicate tokens in the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", seq({ str("ab"), i_sym(0), str("ab") }) },
},
set<Symbol>(),
set<char>()
});
it("preserves the separator characters in the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", str("ab") },
},
set<Symbol>(),
{ 'x', 'y', 'z' }
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0), i_aux_token(0) }) }
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, Equals(rule_list({
{ "'ab'", str("ab") },
})))
});
AssertThat(result.second.separators, Equals(set<char>({ 'x', 'y', 'z' })));
it("preserves the separator characters in the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", str("ab") },
},
set<Symbol>(),
{ 'x', 'y', 'z' }
});
describe("when an entire rule can be extracted", [&]() {
it("moves the rule the lexical grammar when possible and updates referencing symbols", [&]() {
auto result = extract_tokens(InternedGrammar{
{
{ "rule_A", i_sym(1) },
{ "rule_B", pattern("a|b") },
{ "rule_C", token(seq({ str("a"), str("b") })) },
},
set<Symbol>(),
set<char>()
});
AssertThat(result.second.separators, Equals(set<char>({ 'x', 'y', 'z' })));
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", i_token(0) }
})));
AssertThat(result.first.aux_rules, IsEmpty());
AssertThat(result.second.rules, Equals(rule_list({
{ "rule_B", pattern("a|b") },
{ "rule_C", token(seq({ str("a"), str("b") })) },
})));
AssertThat(result.second.aux_rules, IsEmpty());
});
describe("when an entire rule can be extracted", [&]() {
it("moves the rule the lexical grammar when possible and updates referencing symbols", [&]() {
auto result = extract_tokens(InternedGrammar{
{
{ "rule_A", i_sym(1) },
{ "rule_B", pattern("a|b") },
{ "rule_C", token(seq({ str("a"), str("b") })) },
},
set<Symbol>(),
set<char>()
});
it("updates symbols whose indices need to change due to deleted rules", [&]() {
auto result = extract_tokens(InternedGrammar{
{
{ "rule_A", str("ab") },
{ "rule_B", i_sym(0) },
{ "rule_C", i_sym(1) },
},
set<Symbol>(),
set<char>()
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_B", i_token(0) },
{ "rule_C", i_sym(0) },
})));
AssertThat(result.first.aux_rules, IsEmpty());
AssertThat(result.second.rules, Equals(rule_list({
{ "rule_A", str("ab") },
})));
AssertThat(result.second.aux_rules, IsEmpty());
});
it("updates the grammar's ubiquitous_tokens", [&]() {
auto result = extract_tokens(InternedGrammar{
{
{ "rule_A", str("ab") },
{ "rule_B", i_sym(0) },
{ "rule_C", i_sym(1) },
},
{ Symbol(0) },
set<char>()
});
AssertThat(result.first.ubiquitous_tokens, Equals(set<Symbol>({
{ Symbol(0, SymbolOptionToken) }
})));
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", i_token(0) }
})));
AssertThat(result.first.aux_rules, IsEmpty());
AssertThat(result.second.rules, Equals(rule_list({
{ "rule_B", pattern("a|b") },
{ "rule_C", token(seq({ str("a"), str("b") })) },
})));
AssertThat(result.second.aux_rules, IsEmpty());
});
it("updates symbols whose indices need to change due to deleted rules", [&]() {
auto result = extract_tokens(InternedGrammar{
{
{ "rule_A", str("ab") },
{ "rule_B", i_sym(0) },
{ "rule_C", i_sym(1) },
},
set<Symbol>(),
set<char>()
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_B", i_token(0) },
{ "rule_C", i_sym(0) },
})));
AssertThat(result.first.aux_rules, IsEmpty());
AssertThat(result.second.rules, Equals(rule_list({
{ "rule_A", str("ab") },
})));
AssertThat(result.second.aux_rules, IsEmpty());
});
it("updates the grammar's ubiquitous_tokens", [&]() {
auto result = extract_tokens(InternedGrammar{
{
{ "rule_A", str("ab") },
{ "rule_B", i_sym(0) },
{ "rule_C", i_sym(1) },
},
{ Symbol(0) },
set<char>()
});
AssertThat(result.first.ubiquitous_tokens, Equals(set<Symbol>({
{ Symbol(0, SymbolOptionToken) }
})));
});
});
});
END_TEST

84
spec/compiler/prepare_grammar/intern_symbols_spec.cc
View file

@ -11,59 +11,59 @@ using namespace rules;
using prepare_grammar::intern_symbols;
describe("interning symbols in a grammar", []() {
it("replaces named symbols with numerically-indexed symbols", [&]() {
Grammar grammar({
{ "x", choice({ sym("y"), sym("z") }) },
{ "y", sym("z") },
{ "z", str("stuff") }
});
auto result = intern_symbols(grammar);
AssertThat(result.second, Equals((GrammarError *)nullptr));
AssertThat(result.first.rules, Equals(rule_list({
{ "x", choice({ i_sym(1), i_sym(2) }) },
{ "y", i_sym(2) },
{ "z", str("stuff") },
})));
it("replaces named symbols with numerically-indexed symbols", [&]() {
Grammar grammar({
{ "x", choice({ sym("y"), sym("z") }) },
{ "y", sym("z") },
{ "z", str("stuff") }
});
describe("when there are symbols that reference undefined rules", [&]() {
it("returns an error", []() {
Grammar grammar({
{ "x", sym("y") },
});
auto result = intern_symbols(grammar);
auto result = intern_symbols(grammar);
AssertThat(result.second, Equals((GrammarError *)nullptr));
AssertThat(result.first.rules, Equals(rule_list({
{ "x", choice({ i_sym(1), i_sym(2) }) },
{ "y", i_sym(2) },
{ "z", str("stuff") },
})));
});
AssertThat(result.second->message, Equals("Undefined rule 'y'"));
});
describe("when there are symbols that reference undefined rules", [&]() {
it("returns an error", []() {
Grammar grammar({
{ "x", sym("y") },
});
auto result = intern_symbols(grammar);
AssertThat(result.second->message, Equals("Undefined rule 'y'"));
});
});
it("translates the grammar's optional 'ubiquitous_tokens' to numerical symbols", [&]() {
auto grammar = Grammar({
{ "x", choice({ sym("y"), sym("z") }) },
{ "y", sym("z") },
{ "z", str("stuff") }
}).ubiquitous_tokens({ "z" });
it("translates the grammar's optional 'ubiquitous_tokens' to numerical symbols", [&]() {
auto grammar = Grammar({
{ "x", choice({ sym("y"), sym("z") }) },
{ "y", sym("z") },
{ "z", str("stuff") }
}).ubiquitous_tokens({ "z" });
auto result = intern_symbols(grammar);
auto result = intern_symbols(grammar);
AssertThat(result.second, Equals((GrammarError *)nullptr));
AssertThat(result.first.ubiquitous_tokens, Equals(set<Symbol>({
Symbol(2)
})));
});
AssertThat(result.second, Equals((GrammarError *)nullptr));
AssertThat(result.first.ubiquitous_tokens, Equals(set<Symbol>({
Symbol(2)
})));
});
it("preserves the grammar's separator character set", [&]() {
auto grammar = Grammar({
{ "z", str("stuff") }
}).separators({ 'x', 'y' });
it("preserves the grammar's separator character set", [&]() {
auto grammar = Grammar({
{ "z", str("stuff") }
}).separators({ 'x', 'y' });
auto result = intern_symbols(grammar);
auto result = intern_symbols(grammar);
AssertThat(result.first.separators, Equals(set<char>({ 'x', 'y' })))
});
AssertThat(result.first.separators, Equals(set<char>({ 'x', 'y' })))
});
});
END_TEST

398
spec/compiler/prepare_grammar/parse_regex_spec.cc
View file

@ -7,225 +7,225 @@ using namespace rules;
using prepare_grammar::parse_regex;
describe("parsing regex patterns", []() {
struct ValidInputRow {
string description;
string pattern;
rule_ptr rule;
};
struct ValidInputRow {
string description;
string pattern;
rule_ptr rule;
};
vector<ValidInputRow> valid_inputs = {
{
"character sets",
"[aAeE]",
character({ 'a', 'A', 'e', 'E' })
},
vector<ValidInputRow> valid_inputs = {
{
"character sets",
"[aAeE]",
character({ 'a', 'A', 'e', 'E' })
},
{
"'.' characters as wildcards",
".",
CharacterSet({'\n'}).complement().copy()
},
{
"'.' characters as wildcards",
".",
CharacterSet({'\n'}).complement().copy()
},
{
"character classes",
"\\w-\\d",
seq({
character({ {'a', 'z'}, {'A', 'Z'}, {'0', '9'} }),
character({ '-' }),
character({ {'0', '9'} }) })
},
{
"character classes",
"\\w-\\d",
seq({
character({ {'a', 'z'}, {'A', 'Z'}, {'0', '9'} }),
character({ '-' }),
character({ {'0', '9'} }) })
},
{
"choices",
"ab|cd|ef",
choice({
seq({
character({ 'a' }),
character({ 'b' }),
}),
seq({
character({ 'c' }),
character({ 'd' })
}),
seq({
character({ 'e' }),
character({ 'f' })
})
})
},
{
"choices",
"ab|cd|ef",
choice({
seq({
character({ 'a' }),
character({ 'b' }),
}),
seq({
character({ 'c' }),
character({ 'd' })
}),
seq({
character({ 'e' }),
character({ 'f' })
})
})
},
{
"simple sequences",
"abc",
seq({
character({ 'a' }),
character({ 'b' }),
character({ 'c' }) })
},
{
"simple sequences",
"abc",
seq({
character({ 'a' }),
character({ 'b' }),
character({ 'c' }) })
},
{
"character ranges",
"[12a-dA-D3]",
character({ {'1', '3'}, {'a', 'd'}, { 'A', 'D' }, })
},
{
"character ranges",
"[12a-dA-D3]",
character({ {'1', '3'}, {'a', 'd'}, { 'A', 'D' }, })
},
{
"negated characters",
"[^a\\d]",
character({ {'a'}, {'0', '9'} }, false)
},
{
"negated characters",
"[^a\\d]",
character({ {'a'}, {'0', '9'} }, false)
},
{
"backslashes",
"\\\\",
character({ '\\' })
},
{
"backslashes",
"\\\\",
character({ '\\' })
},
{
"character groups in sequences",
"x([^x]|\\\\x)*x",
seq({
character({ 'x' }),
repeat(choice({
character({ 'x' }, false),
seq({ character({ '\\' }), character({ 'x' }) })
})),
character({ 'x' })
})
},
{
"character groups in sequences",
"x([^x]|\\\\x)*x",
seq({
character({ 'x' }),
repeat(choice({
character({ 'x' }, false),
seq({ character({ '\\' }), character({ 'x' }) })
})),
character({ 'x' })
})
},
{
"choices in sequences",
"(a|b)cd",
seq({
choice({
character({ 'a' }),
character({ 'b' }),
}),
character({ 'c' }),
character({ 'd' })
})
},
{
"choices in sequences",
"(a|b)cd",
seq({
choice({
character({ 'a' }),
character({ 'b' }),
}),
character({ 'c' }),
character({ 'd' })
})
},
{
"escaped parentheses",
"a\\(b",
seq({
character({ 'a' }),
character({ '(' }),
character({ 'b' })
})
},
{
"escaped parentheses",
"a\\(b",
seq({
character({ 'a' }),
character({ '(' }),
character({ 'b' })
})
},
{
"escaped periods",
"a\\.",
seq({
character({ 'a' }),
character({ '.' })
})
},
{
"escaped periods",
"a\\.",
seq({
character({ 'a' }),
character({ '.' })
})
},
{
"escaped characters",
"\\t\\n\\r",
seq({
character({ '\t' }),
character({ '\n' }),
character({ '\r' }),
})
},
{
"escaped characters",
"\\t\\n\\r",
seq({
character({ '\t' }),
character({ '\n' }),
character({ '\r' }),
})
},
{
"plus repeats",
"(ab)+(cd)+",
seq({
seq({
seq({ character({ 'a' }), character({ 'b' }) }),
repeat(seq({ character({ 'a' }), character({ 'b' }) })),
}),
seq({
seq({ character({ 'c' }), character({ 'd' }) }),
repeat(seq({ character({ 'c' }), character({ 'd' }) })),
}),
})
},
{
"plus repeats",
"(ab)+(cd)+",
seq({
seq({
seq({ character({ 'a' }), character({ 'b' }) }),
repeat(seq({ character({ 'a' }), character({ 'b' }) })),
}),
seq({
seq({ character({ 'c' }), character({ 'd' }) }),
repeat(seq({ character({ 'c' }), character({ 'd' }) })),
}),
})
},
{
"asterix repeats",
"(ab)*(cd)*",
seq({
repeat(seq({ character({ 'a' }), character({ 'b' }) })),
repeat(seq({ character({ 'c' }), character({ 'd' }) })),
})
},
{
"asterix repeats",
"(ab)*(cd)*",
seq({
repeat(seq({ character({ 'a' }), character({ 'b' }) })),
repeat(seq({ character({ 'c' }), character({ 'd' }) })),
})
},
{
"optional rules",
"a(bc)?",
seq({
character({ 'a' }),
choice({
seq({ character({ 'b' }), character({ 'c' }) }),
blank()
})
})
}
};
{
"optional rules",
"a(bc)?",
seq({
character({ 'a' }),
choice({
seq({ character({ 'b' }), character({ 'c' }) }),
blank()
})
})
}
};
struct InvalidInputRow {
string description;
string pattern;
const char *message;
};
struct InvalidInputRow {
string description;
string pattern;
const char *message;
};
vector<InvalidInputRow> invalid_inputs = {
{
"mismatched open parens",
"(a",
"unmatched open paren",
},
{
"mismatched nested open parens",
"((a) (b)",
"unmatched open paren",
},
{
"mismatched close parens",
"a)",
"unmatched close paren",
},
{
"mismatched nested close parens",
"((a) b))",
"unmatched close paren",
},
{
"mismatched brackets for character classes",
"[a",
"unmatched open square bracket",
},
{
"mismatched brackets for character classes",
"a]",
"unmatched close square bracket",
},
};
vector<InvalidInputRow> invalid_inputs = {
{
"mismatched open parens",
"(a",
"unmatched open paren",
},
{
"mismatched nested open parens",
"((a) (b)",
"unmatched open paren",
},
{
"mismatched close parens",
"a)",
"unmatched close paren",
},
{
"mismatched nested close parens",
"((a) b))",
"unmatched close paren",
},
{
"mismatched brackets for character classes",
"[a",
"unmatched open square bracket",
},
{
"mismatched brackets for character classes",
"a]",
"unmatched close square bracket",
},
};
for (auto &row : valid_inputs) {
it(("parses " + row.description).c_str(), [&]() {
auto result = parse_regex(row.pattern);
AssertThat(result.first, EqualsPointer(row.rule));
});
}
for (auto &row : valid_inputs) {
it(("parses " + row.description).c_str(), [&]() {
auto result = parse_regex(row.pattern);
AssertThat(result.first, EqualsPointer(row.rule));
});
}
for (auto &row : invalid_inputs) {
it(("handles invalid regexes with " + row.description).c_str(), [&]() {
auto result = parse_regex(row.pattern);
AssertThat(result.second, !Equals((const GrammarError *)nullptr));
AssertThat(result.second->message, Contains(row.message));
});
}
for (auto &row : invalid_inputs) {
it(("handles invalid regexes with " + row.description).c_str(), [&]() {
auto result = parse_regex(row.pattern);
AssertThat(result.second, !Equals((const GrammarError *)nullptr));
AssertThat(result.second->message, Contains(row.message));
});
}
});
END_TEST

188
spec/compiler/rules/character_set_spec.cc
View file

@ -6,107 +6,107 @@ using namespace rules;
START_TEST
describe("character sets", []() {
unsigned char max_char = 255;
unsigned char max_char = 255;
describe("computing the complement", [&]() {
it("works for the set containing only the null character", [&]() {
CharacterSet set1({ '\0' });
auto set2 = set1.complement();
AssertThat(set2, Equals(CharacterSet({
{ 1, max_char }
})));
AssertThat(set2.complement(), Equals(set1));
});
it("works for single character sets", [&]() {
CharacterSet set1({ 'b' });
auto set2 = set1.complement();
AssertThat(set2, Equals(CharacterSet({
{ 0, 'a' },
{ 'c', max_char },
})));
AssertThat(set2.complement(), Equals(set1));
});
describe("computing the complement", [&]() {
it("works for the set containing only the null character", [&]() {
CharacterSet set1({ '\0' });
auto set2 = set1.complement();
AssertThat(set2, Equals(CharacterSet({
{ 1, max_char }
})));
AssertThat(set2.complement(), Equals(set1));
});
describe("computing unions", [&]() {
it("works for disjoint sets", [&]() {
CharacterSet set({ {'a', 'z'} });
set.add_set(CharacterSet({ {'A', 'Z'} }));
AssertThat(set, Equals(CharacterSet({ {'a', 'z'}, {'A', 'Z'} })));
});
it("works for single character sets", [&]() {
CharacterSet set1({ 'b' });
auto set2 = set1.complement();
AssertThat(set2, Equals(CharacterSet({
{ 0, 'a' },
{ 'c', max_char },
})));
AssertThat(set2.complement(), Equals(set1));
});
});
it("works for sets with adjacent ranges", [&]() {
CharacterSet set({ CharacterRange('a', 'r') });
set.add_set(CharacterSet({ CharacterRange('s', 'z') }));
AssertThat(set, Equals(CharacterSet({ {'a', 'z'} })));
});
it("becomes the complete set when the complement is added", [&]() {
CharacterSet set({ 'c' });
auto complement = set.complement();
set.add_set(complement);
AssertThat(set, Equals(CharacterSet({ {0, max_char} })));
});
it("works when the result becomes a continuous range", []() {
CharacterSet set({ {'a', 'd'}, {'f', 'z'} });
set.add_set(CharacterSet({ {'c', 'g'} }));
AssertThat(set, Equals(CharacterSet({ {'a', 'z'} })));
});
it("does nothing for the set of all characters", [&]() {
CharacterSet set({ 'a' });
set.add_set(set.complement());
AssertThat(set, Equals(CharacterSet({ {'\0', max_char} })));
});
describe("computing unions", [&]() {
it("works for disjoint sets", [&]() {
CharacterSet set({ {'a', 'z'} });
set.add_set(CharacterSet({ {'A', 'Z'} }));
AssertThat(set, Equals(CharacterSet({ {'a', 'z'}, {'A', 'Z'} })));
});
describe("subtracting sets", []() {
CharacterSet intersection;
it("works for disjoint sets", [&]() {
CharacterSet set1({ {'a', 'z'} });
intersection = set1.remove_set(CharacterSet({ {'A', 'Z'} }));
AssertThat(set1, Equals(CharacterSet({ {'a', 'z'} })));
AssertThat(intersection, Equals(CharacterSet()));
});
it("works when one set is a proper subset of the other", [&]() {
CharacterSet set1({ {'a','z'} });
intersection = set1.remove_set(CharacterSet({ {'d', 's'} }));
AssertThat(set1, Equals(CharacterSet({ {'a', 'c'}, {'t', 'z'} })));
AssertThat(intersection, Equals(CharacterSet({ {'d', 's'} })));
});
it("works for a set that overlaps the right side", [&]() {
CharacterSet set1({ {'a','s'} });
intersection = set1.remove_set(CharacterSet({ {'m', 'z'} }));
AssertThat(set1, Equals(CharacterSet({ {'a', 'l'} })));
AssertThat(intersection, Equals(CharacterSet({ {'m', 's'} })));
});
it("works for a set that overlaps the left side", [&]() {
CharacterSet set2({ {'m','z'} });
intersection = set2.remove_set(CharacterSet({ {'a', 's'} }));
AssertThat(set2, Equals(CharacterSet({ {'t', 'z'} })));
AssertThat(intersection, Equals(CharacterSet({ {'m', 's'} })));
});
it("works for sets with multiple ranges", [&]() {
CharacterSet set1({ {'a', 'd'}, {'m', 'z'} });
intersection = set1.remove_set(CharacterSet({ {'c', 'o'}, {'s', 'x'} }));
AssertThat(set1, Equals(CharacterSet({ {'a', 'b'}, {'p', 'r'}, {'y', 'z'} })));
AssertThat(intersection, Equals(CharacterSet({ {'c', 'd'}, {'m', 'o'}, {'s', 'x'} })));
});
it("works when the result is empty", [&]() {
CharacterSet set1({ 'd' });
intersection = set1.remove_set(CharacterSet({ 'a', 'd', 'x' }));
AssertThat(set1, Equals(CharacterSet()));
AssertThat(intersection, Equals(CharacterSet({ 'd' })));
});
it("works for sets with adjacent ranges", [&]() {
CharacterSet set({ CharacterRange('a', 'r') });
set.add_set(CharacterSet({ CharacterRange('s', 'z') }));
AssertThat(set, Equals(CharacterSet({ {'a', 'z'} })));
});
it("becomes the complete set when the complement is added", [&]() {
CharacterSet set({ 'c' });
auto complement = set.complement();
set.add_set(complement);
AssertThat(set, Equals(CharacterSet({ {0, max_char} })));
});
it("works when the result becomes a continuous range", []() {
CharacterSet set({ {'a', 'd'}, {'f', 'z'} });
set.add_set(CharacterSet({ {'c', 'g'} }));
AssertThat(set, Equals(CharacterSet({ {'a', 'z'} })));
});
it("does nothing for the set of all characters", [&]() {
CharacterSet set({ 'a' });
set.add_set(set.complement());
AssertThat(set, Equals(CharacterSet({ {'\0', max_char} })));
});
});
describe("subtracting sets", []() {
CharacterSet intersection;
it("works for disjoint sets", [&]() {
CharacterSet set1({ {'a', 'z'} });
intersection = set1.remove_set(CharacterSet({ {'A', 'Z'} }));
AssertThat(set1, Equals(CharacterSet({ {'a', 'z'} })));
AssertThat(intersection, Equals(CharacterSet()));
});
it("works when one set is a proper subset of the other", [&]() {
CharacterSet set1({ {'a','z'} });
intersection = set1.remove_set(CharacterSet({ {'d', 's'} }));
AssertThat(set1, Equals(CharacterSet({ {'a', 'c'}, {'t', 'z'} })));
AssertThat(intersection, Equals(CharacterSet({ {'d', 's'} })));
});
it("works for a set that overlaps the right side", [&]() {
CharacterSet set1({ {'a','s'} });
intersection = set1.remove_set(CharacterSet({ {'m', 'z'} }));
AssertThat(set1, Equals(CharacterSet({ {'a', 'l'} })));
AssertThat(intersection, Equals(CharacterSet({ {'m', 's'} })));
});
it("works for a set that overlaps the left side", [&]() {
CharacterSet set2({ {'m','z'} });
intersection = set2.remove_set(CharacterSet({ {'a', 's'} }));
AssertThat(set2, Equals(CharacterSet({ {'t', 'z'} })));
AssertThat(intersection, Equals(CharacterSet({ {'m', 's'} })));
});
it("works for sets with multiple ranges", [&]() {
CharacterSet set1({ {'a', 'd'}, {'m', 'z'} });
intersection = set1.remove_set(CharacterSet({ {'c', 'o'}, {'s', 'x'} }));
AssertThat(set1, Equals(CharacterSet({ {'a', 'b'}, {'p', 'r'}, {'y', 'z'} })));
AssertThat(intersection, Equals(CharacterSet({ {'c', 'd'}, {'m', 'o'}, {'s', 'x'} })));
});
it("works when the result is empty", [&]() {
CharacterSet set1({ 'd' });
intersection = set1.remove_set(CharacterSet({ 'a', 'd', 'x' }));
AssertThat(set1, Equals(CharacterSet()));
AssertThat(intersection, Equals(CharacterSet({ 'd' })));
});
});
});
END_TEST