traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix indentation in specs

Browse source
This commit is contained in:
Max Brunsfeld 2014-08-06 13:00:35 -07:00
parent 01571da30d
commit b155994491
34 changed files with 1877 additions and 1878 deletions
Download patch file Download diff file Expand all files Collapse all files

86
spec/compiler/build_tables/build_parse_table_spec.cc
View file

@ -10,61 +10,61 @@ using namespace build_tables;
START_TEST
describe("building parse tables", []() {
SyntaxGrammar parse_grammar({
{ "rule0", choice({ i_sym(1), i_sym(2) }) },
{ "rule1", i_token(0) },
{ "rule2", i_token(1) },
}, {}, { Symbol(2, SymbolOptionToken) });
SyntaxGrammar parse_grammar({
{ "rule0", choice({ i_sym(1), i_sym(2) }) },
{ "rule1", i_token(0) },
{ "rule2", i_token(1) },
}, {}, { Symbol(2, SymbolOptionToken) });
LexicalGrammar lex_grammar({
{ "token0", pattern("[a-c]") },
{ "token1", pattern("[b-d]") },
}, {});
LexicalGrammar lex_grammar({
{ "token0", pattern("[a-c]") },
{ "token1", pattern("[b-d]") },
}, {});
it("first looks for the start rule and its item set closure", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
it("first looks for the start rule and its item set closure", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
AssertThat(result.first.states[0].actions, Equals(map<Symbol, ParseAction>({
// start item
{ Symbol(0), ParseAction::Shift(1, { 0 }) },
AssertThat(result.first.states[0].actions, Equals(map<Symbol, ParseAction>({
// start item
{ Symbol(0), ParseAction::Shift(1, { 0 }) },
// expanded from the item set closure of the start item
{ Symbol(1), ParseAction::Shift(2, { 0 }) },
{ Symbol(2), ParseAction::Shift(2, { 0 }) },
{ Symbol(0, SymbolOptionToken), ParseAction::Shift(3, { 0 }) },
{ Symbol(1, SymbolOptionToken), ParseAction::Shift(4, { 0 }) },
// expanded from the item set closure of the start item
{ Symbol(1), ParseAction::Shift(2, { 0 }) },
{ Symbol(2), ParseAction::Shift(2, { 0 }) },
{ Symbol(0, SymbolOptionToken), ParseAction::Shift(3, { 0 }) },
{ Symbol(1, SymbolOptionToken), ParseAction::Shift(4, { 0 }) },
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
})));
});
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
})));
});
it("ensures that the built-in 'ERROR' symbol is in the symbol set", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
AssertThat(result.first.symbols, Contains(rules::ERROR()));
});
it("ensures that the built-in 'ERROR' symbol is in the symbol set", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
AssertThat(result.first.symbols, Contains(rules::ERROR()));
});
it("accepts the input when EOF occurs after the start rule", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
it("accepts the input when EOF occurs after the start rule", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
AssertThat(result.first.states[1].actions, Equals(map<Symbol, ParseAction>({
{ END_OF_INPUT(), ParseAction::Accept() },
AssertThat(result.first.states[1].actions, Equals(map<Symbol, ParseAction>({
{ END_OF_INPUT(), ParseAction::Accept() },
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
})));
});
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
})));
});
it("reduces a rule once it has been consumed", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
it("reduces a rule once it has been consumed", [&]() {
auto result = build_parse_table(parse_grammar, lex_grammar);
AssertThat(result.first.states[2].actions, Equals(map<Symbol, ParseAction>({
{ END_OF_INPUT(), ParseAction::Reduce(Symbol(0), 1, 0) },
AssertThat(result.first.states[2].actions, Equals(map<Symbol, ParseAction>({
{ END_OF_INPUT(), ParseAction::Reduce(Symbol(0), 1, 0) },
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
})));
});
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
})));
});
});

406
spec/compiler/build_tables/conflict_manager_spec.cc
View file

@ -10,217 +10,217 @@ using namespace build_tables;
START_TEST
describe("resolving parse conflicts", []() {
bool update;
bool update;
SyntaxGrammar parse_grammar({
{ "rule1", seq({ sym("rule2"), sym("token2") }) },
{ "rule2", sym("token1") },
}, {}, set<rules::Symbol>());
SyntaxGrammar parse_grammar({
{ "rule1", seq({ sym("rule2"), sym("token2") }) },
{ "rule2", sym("token1") },
}, {}, set<rules::Symbol>());
LexicalGrammar lex_grammar({
{ "token1", pattern("[a-c]") },
{ "token2", pattern("[b-d]") },
{ "token3", keyword("stuff") },
}, {}, set<char>());
LexicalGrammar lex_grammar({
{ "token1", pattern("[a-c]") },
{ "token2", pattern("[b-d]") },
{ "token3", keyword("stuff") },
}, {}, set<char>());
describe("lexical conflicts", [&]() {
Symbol sym1(0, SymbolOptionToken);
Symbol sym2(1, SymbolOptionToken);
Symbol sym3(2, SymbolOptionToken);
describe("lexical conflicts", [&]() {
Symbol sym1(0, SymbolOptionToken);
Symbol sym2(1, SymbolOptionToken);
Symbol sym3(2, SymbolOptionToken);
LexConflictManager *manager;
LexConflictManager *manager;
before_each([&]() {
manager = new LexConflictManager(lex_grammar);
});
after_each([&]() {
delete manager;
});
it("favors non-errors over lexical errors", [&]() {
update = manager->resolve_lex_action(LexAction::Error(), LexAction::Advance(2, {0}));
AssertThat(update, IsTrue());
update = manager->resolve_lex_action(LexAction::Advance(2, {0}), LexAction::Error());
AssertThat(update, IsFalse());
});
describe("accept-token/advance conflicts", [&]() {
it("prefers the advance", [&]() {
update = manager->resolve_lex_action(LexAction::Accept(sym3, 3), LexAction::Advance(1, { 0 }));
AssertThat(update, IsTrue());
update = manager->resolve_lex_action(LexAction::Advance(1, { 0 }), LexAction::Accept(sym3, 3));
AssertThat(update, IsFalse());
});
});
describe("accept-token/accept-token conflicts", [&]() {
describe("when one token has a higher precedence than the other", [&]() {
it("prefers the token with the higher precedence", [&]() {
update = manager->resolve_lex_action(LexAction::Accept(sym3, 2), LexAction::Accept(sym2, 0));
AssertThat(update, IsFalse());
update = manager->resolve_lex_action(LexAction::Accept(sym2, 0), LexAction::Accept(sym3, 2));
AssertThat(update, IsTrue());
});
});
describe("when both tokens have the same precedence", [&]() {
it("prefers the token listed earlier in the grammar", [&]() {
update = manager->resolve_lex_action(LexAction::Accept(sym1, 0), LexAction::Accept(sym2, 0));
AssertThat(update, IsFalse());
update = manager->resolve_lex_action(LexAction::Accept(sym2, 0), LexAction::Accept(sym1, 0));
AssertThat(update, IsTrue());
});
});
});
before_each([&]() {
manager = new LexConflictManager(lex_grammar);
});
describe("syntactic conflicts", [&]() {
Symbol sym1(0);
Symbol sym2(1);
ParseConflictManager *manager;
before_each([&]() {
manager = new ParseConflictManager(parse_grammar, lex_grammar);
});
after_each([&]() {
delete manager;
});
it("favors non-errors over parse errors", [&]() {
update = manager->resolve_parse_action(sym1, ParseAction::Error(), ParseAction::Shift(2, { 0 }));
AssertThat(update, IsTrue());
update = manager->resolve_parse_action(sym1, ParseAction::Shift(2, { 0 }), ParseAction::Error());
AssertThat(update, IsFalse());
});
describe("shift/reduce conflicts", [&]() {
describe("when the shift has higher precedence", [&]() {
ParseAction shift = ParseAction::Shift(2, { 3 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 1);
it("does not record a conflict", [&]() {
manager->resolve_parse_action(sym1, shift, reduce);
manager->resolve_parse_action(sym1, reduce, shift);
AssertThat(manager->conflicts(), IsEmpty());
});
it("favors the shift", [&]() {
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsTrue());
});
});
describe("when the reduce has higher precedence", [&]() {
ParseAction shift = ParseAction::Shift(2, { 1 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 3);
it("does not record a conflict", [&]() {
manager->resolve_parse_action(sym1, reduce, shift);
manager->resolve_parse_action(sym1, shift, reduce);
AssertThat(manager->conflicts(), IsEmpty());
});
it("favors the reduce", [&]() {
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsTrue());
});
});
describe("when the precedences are equal", [&]() {
ParseAction shift = ParseAction::Shift(2, { 0 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 0);
it("records a conflict", [&]() {
manager->resolve_parse_action(sym1, reduce, shift);
manager->resolve_parse_action(sym1, shift, reduce);
AssertThat(manager->conflicts(), Equals(vector<Conflict>({
Conflict("rule1: shift (precedence 0) / reduce rule2 (precedence 0)")
})));
});
it("favors the shift", [&]() {
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsTrue());
});
describe("when the symbols is a built-in symbol", [&]() {
it("records a conflict", [&]() {
manager->resolve_parse_action(rules::ERROR(), reduce, shift);
AssertThat(manager->conflicts()[0], Equals(
Conflict("ERROR: shift (precedence 0) / reduce rule2 (precedence 0)")
));
manager->resolve_parse_action(rules::END_OF_INPUT(), reduce, shift);
AssertThat(manager->conflicts()[1], Equals(
Conflict("END_OF_INPUT: shift (precedence 0) / reduce rule2 (precedence 0)")
));
});
});
});
describe("when the shift has conflicting precedences compared to the reduce", [&]() {
ParseAction shift = ParseAction::Shift(2, { 0, 1, 3 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 2);
it("records a conflict", [&]() {
manager->resolve_parse_action(sym1, reduce, shift);
manager->resolve_parse_action(sym1, shift, reduce);
AssertThat(manager->conflicts(), Equals(vector<Conflict>({
Conflict("rule1: shift (precedence 0, 1, 3) / reduce rule2 (precedence 2)")
})));
});
it("favors the shift", [&]() {
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsTrue());
});
});
});
describe("reduce/reduce conflicts", [&]() {
describe("when one action has higher precedence", [&]() {
ParseAction left = ParseAction::Reduce(sym2, 1, 0);
ParseAction right = ParseAction::Reduce(sym2, 1, 3);
it("favors that action", [&]() {
AssertThat(manager->resolve_parse_action(sym1, left, right), IsTrue());
AssertThat(manager->resolve_parse_action(sym1, right, left), IsFalse());
});
it("does not record a conflict", [&]() {
manager->resolve_parse_action(sym1, left, right);
manager->resolve_parse_action(sym1, right, left);
AssertThat(manager->conflicts(), IsEmpty());
});
});
describe("when the actions have the same precedence", [&]() {
ParseAction left = ParseAction::Reduce(sym1, 1, 0);
ParseAction right = ParseAction::Reduce(sym2, 1, 0);
it("favors the symbol listed earlier in the grammar", [&]() {
AssertThat(manager->resolve_parse_action(sym1, right, left), IsTrue());
AssertThat(manager->resolve_parse_action(sym1, left, right), IsFalse());
});
it("records a conflict", [&]() {
manager->resolve_parse_action(sym1, left, right);
manager->resolve_parse_action(sym1, right, left);
AssertThat(manager->conflicts(), Equals(vector<Conflict>({
Conflict("rule1: reduce rule2 (precedence 0) / reduce rule1 (precedence 0)"),
Conflict("rule1: reduce rule1 (precedence 0) / reduce rule2 (precedence 0)")
})));
});
});
});
after_each([&]() {
delete manager;
});
it("favors non-errors over lexical errors", [&]() {
update = manager->resolve_lex_action(LexAction::Error(), LexAction::Advance(2, {0}));
AssertThat(update, IsTrue());
update = manager->resolve_lex_action(LexAction::Advance(2, {0}), LexAction::Error());
AssertThat(update, IsFalse());
});
describe("accept-token/advance conflicts", [&]() {
it("prefers the advance", [&]() {
update = manager->resolve_lex_action(LexAction::Accept(sym3, 3), LexAction::Advance(1, { 0 }));
AssertThat(update, IsTrue());
update = manager->resolve_lex_action(LexAction::Advance(1, { 0 }), LexAction::Accept(sym3, 3));
AssertThat(update, IsFalse());
});
});
describe("accept-token/accept-token conflicts", [&]() {
describe("when one token has a higher precedence than the other", [&]() {
it("prefers the token with the higher precedence", [&]() {
update = manager->resolve_lex_action(LexAction::Accept(sym3, 2), LexAction::Accept(sym2, 0));
AssertThat(update, IsFalse());
update = manager->resolve_lex_action(LexAction::Accept(sym2, 0), LexAction::Accept(sym3, 2));
AssertThat(update, IsTrue());
});
});
describe("when both tokens have the same precedence", [&]() {
it("prefers the token listed earlier in the grammar", [&]() {
update = manager->resolve_lex_action(LexAction::Accept(sym1, 0), LexAction::Accept(sym2, 0));
AssertThat(update, IsFalse());
update = manager->resolve_lex_action(LexAction::Accept(sym2, 0), LexAction::Accept(sym1, 0));
AssertThat(update, IsTrue());
});
});
});
});
describe("syntactic conflicts", [&]() {
Symbol sym1(0);
Symbol sym2(1);
ParseConflictManager *manager;
before_each([&]() {
manager = new ParseConflictManager(parse_grammar, lex_grammar);
});
after_each([&]() {
delete manager;
});
it("favors non-errors over parse errors", [&]() {
update = manager->resolve_parse_action(sym1, ParseAction::Error(), ParseAction::Shift(2, { 0 }));
AssertThat(update, IsTrue());
update = manager->resolve_parse_action(sym1, ParseAction::Shift(2, { 0 }), ParseAction::Error());
AssertThat(update, IsFalse());
});
describe("shift/reduce conflicts", [&]() {
describe("when the shift has higher precedence", [&]() {
ParseAction shift = ParseAction::Shift(2, { 3 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 1);
it("does not record a conflict", [&]() {
manager->resolve_parse_action(sym1, shift, reduce);
manager->resolve_parse_action(sym1, reduce, shift);
AssertThat(manager->conflicts(), IsEmpty());
});
it("favors the shift", [&]() {
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsTrue());
});
});
describe("when the reduce has higher precedence", [&]() {
ParseAction shift = ParseAction::Shift(2, { 1 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 3);
it("does not record a conflict", [&]() {
manager->resolve_parse_action(sym1, reduce, shift);
manager->resolve_parse_action(sym1, shift, reduce);
AssertThat(manager->conflicts(), IsEmpty());
});
it("favors the reduce", [&]() {
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsTrue());
});
});
describe("when the precedences are equal", [&]() {
ParseAction shift = ParseAction::Shift(2, { 0 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 0);
it("records a conflict", [&]() {
manager->resolve_parse_action(sym1, reduce, shift);
manager->resolve_parse_action(sym1, shift, reduce);
AssertThat(manager->conflicts(), Equals(vector<Conflict>({
Conflict("rule1: shift (precedence 0) / reduce rule2 (precedence 0)")
})));
});
it("favors the shift", [&]() {
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsTrue());
});
describe("when the symbols is a built-in symbol", [&]() {
it("records a conflict", [&]() {
manager->resolve_parse_action(rules::ERROR(), reduce, shift);
AssertThat(manager->conflicts()[0], Equals(
Conflict("ERROR: shift (precedence 0) / reduce rule2 (precedence 0)")
));
manager->resolve_parse_action(rules::END_OF_INPUT(), reduce, shift);
AssertThat(manager->conflicts()[1], Equals(
Conflict("END_OF_INPUT: shift (precedence 0) / reduce rule2 (precedence 0)")
));
});
});
});
describe("when the shift has conflicting precedences compared to the reduce", [&]() {
ParseAction shift = ParseAction::Shift(2, { 0, 1, 3 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 2);
it("records a conflict", [&]() {
manager->resolve_parse_action(sym1, reduce, shift);
manager->resolve_parse_action(sym1, shift, reduce);
AssertThat(manager->conflicts(), Equals(vector<Conflict>({
Conflict("rule1: shift (precedence 0, 1, 3) / reduce rule2 (precedence 2)")
})));
});
it("favors the shift", [&]() {
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsTrue());
});
});
});
describe("reduce/reduce conflicts", [&]() {
describe("when one action has higher precedence", [&]() {
ParseAction left = ParseAction::Reduce(sym2, 1, 0);
ParseAction right = ParseAction::Reduce(sym2, 1, 3);
it("favors that action", [&]() {
AssertThat(manager->resolve_parse_action(sym1, left, right), IsTrue());
AssertThat(manager->resolve_parse_action(sym1, right, left), IsFalse());
});
it("does not record a conflict", [&]() {
manager->resolve_parse_action(sym1, left, right);
manager->resolve_parse_action(sym1, right, left);
AssertThat(manager->conflicts(), IsEmpty());
});
});
describe("when the actions have the same precedence", [&]() {
ParseAction left = ParseAction::Reduce(sym1, 1, 0);
ParseAction right = ParseAction::Reduce(sym2, 1, 0);
it("favors the symbol listed earlier in the grammar", [&]() {
AssertThat(manager->resolve_parse_action(sym1, right, left), IsTrue());
AssertThat(manager->resolve_parse_action(sym1, left, right), IsFalse());
});
it("records a conflict", [&]() {
manager->resolve_parse_action(sym1, left, right);
manager->resolve_parse_action(sym1, right, left);
AssertThat(manager->conflicts(), Equals(vector<Conflict>({
Conflict("rule1: reduce rule2 (precedence 0) / reduce rule1 (precedence 0)"),
Conflict("rule1: reduce rule1 (precedence 0) / reduce rule2 (precedence 0)")
})));
});
});
});
});
});
END_TEST

155
spec/compiler/build_tables/first_set_spec.cc
View file

@ -3,99 +3,98 @@
#include "compiler/build_tables/first_set.h"
#include "compiler/rules/metadata.h"
using std::set;
using namespace build_tables;
using namespace rules;
START_TEST
describe("computing FIRST sets", []() {
const SyntaxGrammar null_grammar;
const SyntaxGrammar null_grammar;
describe("for a sequence AB", [&]() {
it("ignores B when A cannot be blank", [&]() {
auto rule = seq({ i_token(0), i_token(1) });
describe("for a sequence AB", [&]() {
it("ignores B when A cannot be blank", [&]() {
auto rule = seq({ i_token(0), i_token(1) });
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
})));
});
it("includes FIRST(B) when A can be blank", [&]() {
auto rule = seq({
choice({
i_token(0),
blank() }),
i_token(1) });
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(1, SymbolOptionToken)
})));
});
it("includes FIRST(A's right hand side) when A is a non-terminal", [&]() {
auto rule = choice({
seq({
i_token(0),
i_token(1) }),
i_sym(0) });
SyntaxGrammar grammar({
{ "rule0", seq({
i_token(2),
i_token(3),
i_token(4) }) }
}, {});
AssertThat(first_set(rule, grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(2, SymbolOptionToken),
})));
});
it("includes FIRST(B) when A is a non-terminal and its expansion can be blank", [&]() {
auto rule = seq({
i_sym(0),
i_token(1) });
SyntaxGrammar grammar({
{ "rule0", choice({
i_token(0),
blank() }) }
}, {});
AssertThat(first_set(rule, grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(1, SymbolOptionToken),
})));
});
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
})));
});
describe("when there are left-recursive rules", [&]() {
it("terminates", [&]() {
SyntaxGrammar grammar({
{ "rule0", choice({
seq({ i_sym(0), i_token(10) }),
i_token(11),
}) },
}, {});
it("includes FIRST(B) when A can be blank", [&]() {
auto rule = seq({
choice({
i_token(0),
blank() }),
i_token(1) });
auto rule = i_sym(0);
AssertThat(first_set(rule, grammar), Equals(set<Symbol>({
Symbol(11, SymbolOptionToken)
})));
});
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(1, SymbolOptionToken)
})));
});
it("ignores metadata rules", [&]() {
auto rule = make_shared<Metadata>(i_token(3), map<rules::MetadataKey, int>());
it("includes FIRST(A's right hand side) when A is a non-terminal", [&]() {
auto rule = choice({
seq({
i_token(0),
i_token(1) }),
i_sym(0) });
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol(3, SymbolOptionToken),
})));
SyntaxGrammar grammar({
{ "rule0", seq({
i_token(2),
i_token(3),
i_token(4) }) }
}, {});
AssertThat(first_set(rule, grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(2, SymbolOptionToken),
})));
});
it("includes FIRST(B) when A is a non-terminal and its expansion can be blank", [&]() {
auto rule = seq({
i_sym(0),
i_token(1) });
SyntaxGrammar grammar({
{ "rule0", choice({
i_token(0),
blank() }) }
}, {});
AssertThat(first_set(rule, grammar), Equals(set<Symbol>({
Symbol(0, SymbolOptionToken),
Symbol(1, SymbolOptionToken),
})));
});
});
describe("when there are left-recursive rules", [&]() {
it("terminates", [&]() {
SyntaxGrammar grammar({
{ "rule0", choice({
seq({ i_sym(0), i_token(10) }),
i_token(11),
}) },
}, {});
auto rule = i_sym(0);
AssertThat(first_set(rule, grammar), Equals(set<Symbol>({
Symbol(11, SymbolOptionToken)
})));
});
});
it("ignores metadata rules", [&]() {
auto rule = make_shared<Metadata>(i_token(3), map<rules::MetadataKey, int>());
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol(3, SymbolOptionToken),
})));
});
});
END_TEST

68
spec/compiler/build_tables/get_metadata_spec.cc
View file

@ -8,44 +8,44 @@ using namespace build_tables;
START_TEST
describe("getting metadata for rules", []() {
MetadataKey key1 = MetadataKey(100);
MetadataKey key2 = MetadataKey(101);
rule_ptr rule;
MetadataKey key1 = MetadataKey(100);
MetadataKey key2 = MetadataKey(101);
rule_ptr rule;
describe("when given a metadata rule", [&]() {
before_each([&]() {
rule = make_shared<Metadata>(sym("x"), map<MetadataKey, int>({
{ key1, 1 },
{ key2, 2 },
}));
});
it("returns the value for the given key", [&]() {
AssertThat(get_metadata(rule, key1), Equals(1));
AssertThat(get_metadata(rule, key2), Equals(2));
});
it("returns 0 if the rule does not have the key", [&]() {
AssertThat(get_metadata(rule, MetadataKey(0)), Equals(0));
});
describe("when the rule contains another metadata rule", [&]() {
it("also gets metadata from the inner metadata rule", [&]() {
rule = make_shared<Metadata>(make_shared<Metadata>(sym("x"), map<MetadataKey, int>({
{ key1, 1 }
})), map<MetadataKey, int>());
AssertThat(get_metadata(rule, key1), Equals(1));
});
});
describe("when given a metadata rule", [&]() {
before_each([&]() {
rule = make_shared<Metadata>(sym("x"), map<MetadataKey, int>({
{ key1, 1 },
{ key2, 2 },
}));
});
describe("when given a non-metadata rule", [&]() {
it("returns 0", [&]() {
auto rule = sym("x");
AssertThat(get_metadata(rule, key1), Equals(0));
});
it("returns the value for the given key", [&]() {
AssertThat(get_metadata(rule, key1), Equals(1));
AssertThat(get_metadata(rule, key2), Equals(2));
});
it("returns 0 if the rule does not have the key", [&]() {
AssertThat(get_metadata(rule, MetadataKey(0)), Equals(0));
});
describe("when the rule contains another metadata rule", [&]() {
it("also gets metadata from the inner metadata rule", [&]() {
rule = make_shared<Metadata>(make_shared<Metadata>(sym("x"), map<MetadataKey, int>({
{ key1, 1 }
})), map<MetadataKey, int>());
AssertThat(get_metadata(rule, key1), Equals(1));
});
});
});
describe("when given a non-metadata rule", [&]() {
it("returns 0", [&]() {
auto rule = sym("x");
AssertThat(get_metadata(rule, key1), Equals(0));
});
});
});
END_TEST

34
spec/compiler/build_tables/item_set_closure_spec.cc
View file

@ -9,25 +9,25 @@ using namespace rules;
START_TEST
describe("computing closures of item sets", []() {
SyntaxGrammar grammar({
{ "E", seq({
i_sym(1),
i_token(11) }) },
{ "T", seq({
i_token(12),
i_token(13) }) },
}, {});
SyntaxGrammar grammar({
{ "E", seq({
i_sym(1),
i_token(11) }) },
{ "T", seq({
i_token(12),
i_token(13) }) },
}, {});
it("adds items at the beginnings of referenced rules", [&]() {
ParseItemSet item_set = item_set_closure(ParseItem(Symbol(0), grammar.rule(Symbol(0)), 0),
{ Symbol(10, SymbolOptionToken) },
grammar);
it("adds items at the beginnings of referenced rules", [&]() {
ParseItemSet item_set = item_set_closure(ParseItem(Symbol(0), grammar.rule(Symbol(0)), 0),
{ Symbol(10, SymbolOptionToken) },
grammar);
AssertThat(item_set, Equals(ParseItemSet({
{ ParseItem(Symbol(1), grammar.rule(Symbol(1)), 0), { Symbol(11, SymbolOptionToken) } },
{ ParseItem(Symbol(0), grammar.rule(Symbol(0)), 0), { Symbol(10, SymbolOptionToken) } },
})));
});
AssertThat(item_set, Equals(ParseItemSet({
{ ParseItem(Symbol(1), grammar.rule(Symbol(1)), 0), { Symbol(11, SymbolOptionToken) } },
{ ParseItem(Symbol(0), grammar.rule(Symbol(0)), 0), { Symbol(10, SymbolOptionToken) } },
})));
});
});
END_TEST

60
spec/compiler/build_tables/item_set_transitions_spec.cc
View file

@ -8,43 +8,43 @@ using namespace build_tables;
START_TEST
describe("lexical item set transitions", []() {
describe("when two items in the set have transitions on the same character", [&]() {
it("merges the transitions by computing the union of the two item sets", [&]() {
LexItemSet set1({
LexItem(Symbol(1), character({ {'a', 'f'} })),
LexItem(Symbol(2), character({ {'e', 'x'} })) });
describe("when two items in the set have transitions on the same character", [&]() {
it("merges the transitions by computing the union of the two item sets", [&]() {
LexItemSet set1({
LexItem(Symbol(1), character({ {'a', 'f'} })),
LexItem(Symbol(2), character({ {'e', 'x'} })) });
AssertThat(char_transitions(set1), Equals(map<CharacterSet, LexItemSet>({
{ CharacterSet({ {'a', 'd'} }), LexItemSet({
LexItem(Symbol(1), blank()) }) },
{ CharacterSet({ {'e', 'f'} }), LexItemSet({
LexItem(Symbol(1), blank()),
LexItem(Symbol(2), blank()) }) },
{ CharacterSet({ {'g', 'x'} }), LexItemSet({
LexItem(Symbol(2), blank()) }) },
})));
});
AssertThat(char_transitions(set1), Equals(map<CharacterSet, LexItemSet>({
{ CharacterSet({ {'a', 'd'} }), LexItemSet({
LexItem(Symbol(1), blank()) }) },
{ CharacterSet({ {'e', 'f'} }), LexItemSet({
LexItem(Symbol(1), blank()),
LexItem(Symbol(2), blank()) }) },
{ CharacterSet({ {'g', 'x'} }), LexItemSet({
LexItem(Symbol(2), blank()) }) },
})));
});
});
});
describe("syntactic item set transitions", [&]() {
SyntaxGrammar grammar({
{ "A", blank() },
{ "B", i_token(21) },
}, {}, set<Symbol>());
SyntaxGrammar grammar({
{ "A", blank() },
{ "B", i_token(21) },
}, {}, set<Symbol>());
it("computes the closure of the new item sets", [&]() {
ParseItemSet set1({
{ ParseItem(Symbol(0), seq({ i_token(22), i_sym(1) }), 3), { Symbol(23, SymbolOptionToken) } },
});
AssertThat(sym_transitions(set1, grammar), Equals(map<Symbol, ParseItemSet>({
{ Symbol(22, SymbolOptionToken), ParseItemSet({
{ ParseItem(Symbol(0), i_sym(1), 4), { Symbol(23, SymbolOptionToken) } },
{ ParseItem(Symbol(1), i_token(21), 0), { Symbol(23, SymbolOptionToken) } },
}) },
})));
it("computes the closure of the new item sets", [&]() {
ParseItemSet set1({
{ ParseItem(Symbol(0), seq({ i_token(22), i_sym(1) }), 3), { Symbol(23, SymbolOptionToken) } },
});
AssertThat(sym_transitions(set1, grammar), Equals(map<Symbol, ParseItemSet>({
{ Symbol(22, SymbolOptionToken), ParseItemSet({
{ ParseItem(Symbol(0), i_sym(1), 4), { Symbol(23, SymbolOptionToken) } },
{ ParseItem(Symbol(1), i_token(21), 0), { Symbol(23, SymbolOptionToken) } },
}) },
})));
});
});
END_TEST

56
spec/compiler/build_tables/lex_item_spec.cc
View file

@ -9,36 +9,36 @@ using namespace build_tables;
START_TEST
describe("lex items", []() {
describe("determining if an item is the start of a token", [&]() {
Symbol sym(1);
rule_ptr token_start = make_shared<Metadata>(str("a"), map<MetadataKey, int>({
{ START_TOKEN, 1 }
}));
describe("determining if an item is the start of a token", [&]() {
Symbol sym(1);
rule_ptr token_start = make_shared<Metadata>(str("a"), map<MetadataKey, int>({
{ START_TOKEN, 1 }
}));
it("returns true for rules designated as token starts", [&]() {
LexItem item(sym, token_start);
AssertThat(item.is_token_start(), IsTrue());
});
it("returns false for rules not designated as token starts", [&]() {
AssertThat(LexItem(sym, make_shared<Metadata>(str("a"), map<MetadataKey, int>({
{ START_TOKEN, 0 }
}))).is_token_start(), IsFalse());
AssertThat(LexItem(sym, str("a")).is_token_start(), IsFalse());
});
describe("when given a sequence containing a token start", [&]() {
it("returns true when the rule before the token start may be blank", [&]() {
LexItem item(sym, seq({ repeat(str("a")), token_start }));
AssertThat(item.is_token_start(), IsTrue());
});
it("returns false when the rule before the token start cannot be blank", [&]() {
LexItem item(sym, seq({ str("a"), token_start }));
AssertThat(item.is_token_start(), IsFalse());
});
});
it("returns true for rules designated as token starts", [&]() {
LexItem item(sym, token_start);
AssertThat(item.is_token_start(), IsTrue());
});
it("returns false for rules not designated as token starts", [&]() {
AssertThat(LexItem(sym, make_shared<Metadata>(str("a"), map<MetadataKey, int>({
{ START_TOKEN, 0 }
}))).is_token_start(), IsFalse());
AssertThat(LexItem(sym, str("a")).is_token_start(), IsFalse());
});
describe("when given a sequence containing a token start", [&]() {
it("returns true when the rule before the token start may be blank", [&]() {
LexItem item(sym, seq({ repeat(str("a")), token_start }));
AssertThat(item.is_token_start(), IsTrue());
});
it("returns false when the rule before the token start cannot be blank", [&]() {
LexItem item(sym, seq({ str("a"), token_start }));
AssertThat(item.is_token_start(), IsFalse());
});
});
});
});
END_TEST

106
spec/compiler/build_tables/merge_transitions_spec.cc
View file

@ -7,69 +7,69 @@ using namespace build_tables;
START_TEST
describe("merging character set transitions", []() {
typedef map<CharacterSet, int> int_map;
typedef map<CharacterSet, int> int_map;
auto do_merge = [&](int_map *left, const pair<CharacterSet, int> &new_pair) {
merge_char_transition<int>(left, new_pair, [](int *l, const int *r) {
*l = *l | *r;
});
};
describe("when none of the transitions intersect", [&]() {
it("returns the union of the two sets of transitions", [&]() {
int_map map({
{ CharacterSet({ 'a', 'c' }), 1 },
{ CharacterSet({ 'x', 'y' }), 2 },
{ CharacterSet({ '1', '9' }), 4 },
});
do_merge(&map, { CharacterSet({ ' ' }), 8 });
do_merge(&map, { CharacterSet({ '\t' }), 16 });
AssertThat(map, Equals(int_map({
{ CharacterSet({ 'a', 'c' }), 1 },
{ CharacterSet({ 'x', 'y' }), 2 },
{ CharacterSet({ '1', '9' }), 4 },
{ CharacterSet({ ' ' }), 8 },
{ CharacterSet({ '\t' }), 16 },
})));
});
auto do_merge = [&](int_map *left, const pair<CharacterSet, int> &new_pair) {
merge_char_transition<int>(left, new_pair, [](int *l, const int *r) {
*l = *l | *r;
});
};
describe("when transitions intersect", [&]() {
it("merges the intersecting transitions using the provided function", [&]() {
int_map map({
{ CharacterSet({ {'a', 'f'}, {'A', 'F'} }), 1 },
{ CharacterSet({ {'0', '9'} }), 2 },
});
describe("when none of the transitions intersect", [&]() {
it("returns the union of the two sets of transitions", [&]() {
int_map map({
{ CharacterSet({ 'a', 'c' }), 1 },
{ CharacterSet({ 'x', 'y' }), 2 },
{ CharacterSet({ '1', '9' }), 4 },
});
do_merge(&map, { CharacterSet({ 'c' }), 4 });
do_merge(&map, { CharacterSet({ '3' }), 8 });
do_merge(&map, { CharacterSet({ ' ' }), 8 });
do_merge(&map, { CharacterSet({ '\t' }), 16 });
AssertThat(map, Equals(int_map({
{ CharacterSet({ {'a', 'b'}, {'d', 'f'}, {'A', 'F'} }), 1 },
{ CharacterSet({ {'c'} }), 5 },
{ CharacterSet({ {'0', '2'}, {'4', '9'} }), 2 },
{ CharacterSet({ '3' }), 10 },
})));
});
AssertThat(map, Equals(int_map({
{ CharacterSet({ 'a', 'c' }), 1 },
{ CharacterSet({ 'x', 'y' }), 2 },
{ CharacterSet({ '1', '9' }), 4 },
{ CharacterSet({ ' ' }), 8 },
{ CharacterSet({ '\t' }), 16 },
})));
});
});
describe("when two of the right transitions intersect the same left transition", [&]() {
it("splits the left-hand transition correctly", [&]() {
int_map map1({
{ CharacterSet({ 'a', 'c' }), 1 },
});
describe("when transitions intersect", [&]() {
it("merges the intersecting transitions using the provided function", [&]() {
int_map map({
{ CharacterSet({ {'a', 'f'}, {'A', 'F'} }), 1 },
{ CharacterSet({ {'0', '9'} }), 2 },
});
do_merge(&map1, { CharacterSet({ 'a' }), 2 });
do_merge(&map1, { CharacterSet({ 'c' }), 4 });
do_merge(&map, { CharacterSet({ 'c' }), 4 });
do_merge(&map, { CharacterSet({ '3' }), 8 });
AssertThat(map1, Equals(int_map({
{ CharacterSet({ 'a' }), 3 },
{ CharacterSet({ 'c' }), 5 },
})));
});
AssertThat(map, Equals(int_map({
{ CharacterSet({ {'a', 'b'}, {'d', 'f'}, {'A', 'F'} }), 1 },
{ CharacterSet({ {'c'} }), 5 },
{ CharacterSet({ {'0', '2'}, {'4', '9'} }), 2 },
{ CharacterSet({ '3' }), 10 },
})));
});
});
describe("when two of the right transitions intersect the same left transition", [&]() {
it("splits the left-hand transition correctly", [&]() {
int_map map1({
{ CharacterSet({ 'a', 'c' }), 1 },
});
do_merge(&map1, { CharacterSet({ 'a' }), 2 });
do_merge(&map1, { CharacterSet({ 'c' }), 4 });
AssertThat(map1, Equals(int_map({
{ CharacterSet({ 'a' }), 3 },
{ CharacterSet({ 'c' }), 5 },
})));
});
});
});
END_TEST

124
spec/compiler/build_tables/rule_can_be_blank_spec.cc
View file

@ -9,72 +9,72 @@ using build_tables::rule_can_be_blank;
START_TEST
describe("checking if rules can be blank", [&]() {
rule_ptr rule;
rule_ptr rule;
it("returns false for basic rules", [&]() {
AssertThat(rule_can_be_blank(i_sym(3)), IsFalse());
AssertThat(rule_can_be_blank(str("x")), IsFalse());
AssertThat(rule_can_be_blank(pattern("x")), IsFalse());
it("returns false for basic rules", [&]() {
AssertThat(rule_can_be_blank(i_sym(3)), IsFalse());
AssertThat(rule_can_be_blank(str("x")), IsFalse());
AssertThat(rule_can_be_blank(pattern("x")), IsFalse());
});
it("returns true for blanks", [&]() {
AssertThat(rule_can_be_blank(blank()), IsTrue());
});
it("returns true for repeats", [&]() {
AssertThat(rule_can_be_blank(repeat(str("x"))), IsTrue());
});
it("returns true for choices iff one or more sides can be blank", [&]() {
rule = choice({ sym("x"), blank() });
AssertThat(rule_can_be_blank(rule), IsTrue());
rule = choice({ blank(), sym("x") });
AssertThat(rule_can_be_blank(rule), IsTrue());
rule = choice({ sym("x"), sym("y") });
AssertThat(rule_can_be_blank(rule), IsFalse());
});
it("returns true for sequences iff both sides can be blank", [&]() {
rule = seq({ blank(), str("x") });
AssertThat(rule_can_be_blank(rule), IsFalse());
rule = seq({ str("x"), blank() });
AssertThat(rule_can_be_blank(rule), IsFalse());
rule = seq({ blank(), choice({ sym("x"), blank() }) });
AssertThat(rule_can_be_blank(rule), IsTrue());
});
it("ignores metadata rules", [&]() {
rule = make_shared<rules::Metadata>(blank(), map<rules::MetadataKey, int>());
AssertThat(rule_can_be_blank(rule), IsTrue());
rule = make_shared<rules::Metadata>(sym("one"), map<rules::MetadataKey, int>());
AssertThat(rule_can_be_blank(rule), IsFalse());
});
describe("checking recursively (by expanding non-terminals)", [&]() {
SyntaxGrammar grammar({
{ "A", choice({
seq({ i_sym(0), i_token(11) }),
blank() }) },
{ "B", choice({
seq({ i_sym(1), i_token(12) }),
i_token(13) }) },
}, {}, set<Symbol>());
it("terminates for left-recursive rules that can be blank", [&]() {
rule = i_sym(0);
AssertThat(rule_can_be_blank(rule, grammar), IsTrue());
});
it("returns true for blanks", [&]() {
AssertThat(rule_can_be_blank(blank()), IsTrue());
});
it("returns true for repeats", [&]() {
AssertThat(rule_can_be_blank(repeat(str("x"))), IsTrue());
});
it("returns true for choices iff one or more sides can be blank", [&]() {
rule = choice({ sym("x"), blank() });
AssertThat(rule_can_be_blank(rule), IsTrue());
rule = choice({ blank(), sym("x") });
AssertThat(rule_can_be_blank(rule), IsTrue());
rule = choice({ sym("x"), sym("y") });
AssertThat(rule_can_be_blank(rule), IsFalse());
});
it("returns true for sequences iff both sides can be blank", [&]() {
rule = seq({ blank(), str("x") });
AssertThat(rule_can_be_blank(rule), IsFalse());
rule = seq({ str("x"), blank() });
AssertThat(rule_can_be_blank(rule), IsFalse());
rule = seq({ blank(), choice({ sym("x"), blank() }) });
AssertThat(rule_can_be_blank(rule), IsTrue());
});
it("ignores metadata rules", [&]() {
rule = make_shared<rules::Metadata>(blank(), map<rules::MetadataKey, int>());
AssertThat(rule_can_be_blank(rule), IsTrue());
rule = make_shared<rules::Metadata>(sym("one"), map<rules::MetadataKey, int>());
AssertThat(rule_can_be_blank(rule), IsFalse());
});
describe("checking recursively (by expanding non-terminals)", [&]() {
SyntaxGrammar grammar({
{ "A", choice({
seq({ i_sym(0), i_token(11) }),
blank() }) },
{ "B", choice({
seq({ i_sym(1), i_token(12) }),
i_token(13) }) },
}, {}, set<Symbol>());
it("terminates for left-recursive rules that can be blank", [&]() {
rule = i_sym(0);
AssertThat(rule_can_be_blank(rule, grammar), IsTrue());
});
it("terminates for left-recursive rules that can't be blank", [&]() {
rule = i_sym(1);
AssertThat(rule_can_be_blank(rule, grammar), IsFalse());
});
it("terminates for left-recursive rules that can't be blank", [&]() {
rule = i_sym(1);
AssertThat(rule_can_be_blank(rule, grammar), IsFalse());
});
});
});
END_TEST

329
spec/compiler/build_tables/rule_transitions_spec.cc
View file

@ -9,181 +9,180 @@ using namespace build_tables;
START_TEST
describe("rule transitions", []() {
it("handles symbols", [&]() {
AssertThat(
sym_transitions(i_sym(1)),
Equals(rule_map<Symbol>({
{ Symbol(1), blank() }
})));
});
it("handles symbols", [&]() {
AssertThat(
sym_transitions(i_sym(1)),
Equals(rule_map<Symbol>({
{ Symbol(1), blank() }
})));
});
it("handles choices", [&]() {
AssertThat(
sym_transitions(choice({ i_sym(1), i_sym(2) })),
Equals(rule_map<Symbol>({
{ Symbol(1), blank() },
{ Symbol(2), blank() }
})));
});
it("handles choices", [&]() {
AssertThat(
sym_transitions(choice({ i_sym(1), i_sym(2) })),
Equals(rule_map<Symbol>({
{ Symbol(1), blank() },
{ Symbol(2), blank() }
})));
});
it("handles sequences", [&]() {
AssertThat(
sym_transitions(seq({ i_sym(1), i_sym(2) })),
Equals(rule_map<Symbol>({
{ Symbol(1), i_sym(2) }
})));
});
it("handles sequences", [&]() {
AssertThat(
sym_transitions(seq({ i_sym(1), i_sym(2) })),
Equals(rule_map<Symbol>({
{ Symbol(1), i_sym(2) }
})));
});
it("handles long sequences", [&]() {
AssertThat(
sym_transitions(seq({
it("handles long sequences", [&]() {
AssertThat(
sym_transitions(seq({
i_sym(1),
i_sym(2),
i_sym(3),
i_sym(4)
})),
Equals(rule_map<Symbol>({
{ Symbol(1), seq({ i_sym(2), i_sym(3), i_sym(4) }) }
})));
});
it("handles sequences whose left sides can be blank", [&]() {
AssertThat(
sym_transitions(seq({
choice({
i_sym(1),
i_sym(2),
i_sym(3),
i_sym(4)
})),
Equals(rule_map<Symbol>({
{ Symbol(1), seq({ i_sym(2), i_sym(3), i_sym(4) }) }
})));
});
blank(),
}),
seq({
i_sym(1),
i_sym(2)
})
})), Equals(rule_map<Symbol>({
{ Symbol(1), choice({ seq({ i_sym(1), i_sym(2) }), i_sym(2), }) }
})));
});
it("handles sequences whose left sides can be blank", [&]() {
AssertThat(
sym_transitions(seq({
choice({
i_sym(1),
blank(),
}),
it("handles choices with common starting symbols", [&]() {
AssertThat(
sym_transitions(
choice({
seq({ i_sym(1), i_sym(2) }),
seq({ i_sym(1), i_sym(3) }) })),
Equals(rule_map<Symbol>({
{ Symbol(1), choice({ i_sym(2), i_sym(3) }) }
})));
});
it("handles characters", [&]() {
AssertThat(
char_transitions(character({ '1' })),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ '1' }), blank() }
})));
});
it("handles choices between overlapping character sets", [&]() {
AssertThat(
char_transitions(choice({
seq({
character({ {'a', 's'} }),
sym("x") }),
seq({
character({ { 'm', 'z' } }),
sym("y") }) })),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ {'a','l'} }), sym("x") },
{ CharacterSet({ {'m','s'} }), choice({ sym("x"), sym("y") }) },
{ CharacterSet({ {'t','z'} }), sym("y") },
})));
});
it("handles choices between a subset and a superset of characters", [&]() {
AssertThat(
char_transitions(choice({
seq({
character({ {'a', 'c'} }),
sym("x") }),
seq({
character({ { 'a', 'z' } }),
sym("y") }) })),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ {'a', 'c'} }), choice({ sym("x"), sym("y") }) },
{ CharacterSet({ {'d', 'z'} }), sym("y") },
})));
AssertThat(
char_transitions(choice({
seq({
character({ { 'a', 'z' } }),
sym("x") }),
seq({
character({ {'a', 'c'} }),
sym("y") }) })),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ {'a', 'c'} }), choice({ sym("x"), sym("y") }) },
{ CharacterSet({ {'d', 'z'} }), sym("x") },
})));
});
it("handles blanks", [&]() {
AssertThat(char_transitions(blank()), Equals(rule_map<CharacterSet>({})));
});
it("handles repeats", [&]() {
rule_ptr rule = repeat(seq({ character({ 'a' }), character({ 'b' }) }));
AssertThat(
char_transitions(rule),
Equals(rule_map<CharacterSet>({
{
CharacterSet({ 'a' }),
seq({
i_sym(1),
i_sym(2)
character({ 'b' }),
rule,
})
})), Equals(rule_map<Symbol>({
{ Symbol(1), choice({ seq({ i_sym(1), i_sym(2) }), i_sym(2), }) }
})));
}})));
rule = repeat(character({ 'a' }));
AssertThat(
char_transitions(rule),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ 'a' }), rule }
})));
});
it("preserves metadata", [&]() {
map<MetadataKey, int> metadata_value({
{ PRECEDENCE, 5 }
});
it("handles choices with common starting symbols", [&]() {
AssertThat(
sym_transitions(
choice({
seq({ i_sym(1), i_sym(2) }),
seq({ i_sym(1), i_sym(3) }) })),
Equals(rule_map<Symbol>({
{ Symbol(1), choice({ i_sym(2), i_sym(3) }) }
})));
});
it("handles characters", [&]() {
AssertThat(
char_transitions(character({ '1' })),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ '1' }), blank() }
})));
});
it("handles choices between overlapping character sets", [&]() {
AssertThat(
char_transitions(choice({
seq({
character({ {'a', 's'} }),
sym("x") }),
seq({
character({ { 'm', 'z' } }),
sym("y") }) })),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ {'a','l'} }), sym("x") },
{ CharacterSet({ {'m','s'} }), choice({ sym("x"), sym("y") }) },
{ CharacterSet({ {'t','z'} }), sym("y") },
})));
});
it("handles choices between a subset and a superset of characters", [&]() {
AssertThat(
char_transitions(choice({
seq({
character({ {'a', 'c'} }),
sym("x") }),
seq({
character({ { 'a', 'z' } }),
sym("y") }) })),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ {'a', 'c'} }), choice({ sym("x"), sym("y") }) },
{ CharacterSet({ {'d', 'z'} }), sym("y") },
})));
AssertThat(
char_transitions(choice({
seq({
character({ { 'a', 'z' } }),
sym("x") }),
seq({
character({ {'a', 'c'} }),
sym("y") }) })),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ {'a', 'c'} }), choice({ sym("x"), sym("y") }) },
{ CharacterSet({ {'d', 'z'} }), sym("x") },
})));
});
it("handles blanks", [&]() {
AssertThat(char_transitions(blank()), Equals(rule_map<CharacterSet>({})));
});
it("handles repeats", [&]() {
rule_ptr rule = repeat(seq({ character({ 'a' }), character({ 'b' }) }));
AssertThat(
char_transitions(rule),
Equals(rule_map<CharacterSet>({
{
CharacterSet({ 'a' }),
seq({
character({ 'b' }),
rule,
})
}})));
rule = repeat(character({ 'a' }));
AssertThat(
char_transitions(rule),
Equals(rule_map<CharacterSet>({
{ CharacterSet({ 'a' }), rule }
})));
});
it("preserves metadata", [&]() {
map<MetadataKey, int> metadata_value({
{ PRECEDENCE, 5 }
});
rule_ptr rule = make_shared<Metadata>(seq({ i_sym(1), i_sym(2) }), metadata_value);
AssertThat(
sym_transitions(rule),
Equals(rule_map<Symbol>({
{ Symbol(1), make_shared<Metadata>(i_sym(2), metadata_value)},
})));
});
describe("regression tests (somewhat redundant, should maybe be deleted later)", []() {
it("handles sequences that start with repeating characters", [&]() {
auto rule = seq({
choice({
repeat(character({ '"' }, false)),
blank(),
}),
character({ '"' }),
});
AssertThat(char_transitions(rule), Equals(rule_map<CharacterSet>({
{ CharacterSet({ '"' }).complement(), seq({
repeat(character({ '"' }, false)),
character({ '"' }), }) },
{ CharacterSet({ '"' }), blank() },
})));
});
rule_ptr rule = make_shared<Metadata>(seq({ i_sym(1), i_sym(2) }), metadata_value);
AssertThat(
sym_transitions(rule),
Equals(rule_map<Symbol>({
{ Symbol(1), make_shared<Metadata>(i_sym(2), metadata_value)},
})));
});
describe("regression tests (somewhat redundant, should maybe be deleted later)", []() {
it("handles sequences that start with repeating characters", [&]() {
auto rule = seq({
choice({
repeat(character({ '"' }, false)),
blank(),
}),
character({ '"' }),
});
AssertThat(char_transitions(rule), Equals(rule_map<CharacterSet>({
{ CharacterSet({ '"' }).complement(), seq({
repeat(character({ '"' }, false)),
character({ '"' }), }) },
{ CharacterSet({ '"' }), blank() },
})));
});
});
});
END_TEST

53
spec/compiler/compile_examples.cc
View file

@ -2,44 +2,45 @@
#include <fstream>
static string src_dir() {
const char * dir = getenv("TREESITTER_DIR");
if (!dir) dir = getenv("PWD");
return dir;
const char * dir = getenv("TREESITTER_DIR");
if (!dir) dir = getenv("PWD");
return dir;
}
namespace tree_sitter_examples {
extern const Grammar arithmetic;
extern const Grammar javascript;
extern const Grammar json;
extern const Grammar golang;
}
extern const Grammar arithmetic;
extern const Grammar javascript;
extern const Grammar json;
extern const Grammar golang;
} // namespace tree_sitter_examples
START_TEST
describe("compiling the example grammars", []() {
string example_parser_dir = src_dir() + "/examples/parsers/";
string example_parser_dir = src_dir() + "/examples/parsers/";
auto compile_grammar = [&](const Grammar &grammar, string language) {
it(("compiles the " + language + " grammar").c_str(), [&]() {
ofstream file(example_parser_dir + language + ".c");
auto compile_grammar = [&](const Grammar &grammar, string language) {
it(("compiles the " + language + " grammar").c_str(), [&]() {
ofstream file(example_parser_dir + language + ".c");
auto result = compile(grammar, language);
string code = get<0>(result);
vector<Conflict> conflicts = get<1>(result);
const GrammarError *error = get<2>(result);
auto result = compile(grammar, language);
string code = get<0>(result);
vector<Conflict> conflicts = get<1>(result);
const GrammarError *error = get<2>(result);
AssertThat(error, Equals((GrammarError *)nullptr));
// cout << "\n\nconflicts for " << language << ":\n" << get<1>(result);
AssertThat(error, Equals((GrammarError *)nullptr));
file << get<0>(result);
file.close();
});
};
file << get<0>(result);
file.close();
});
};
compile_grammar(tree_sitter_examples::arithmetic, "arithmetic");
compile_grammar(tree_sitter_examples::json, "json");
compile_grammar(tree_sitter_examples::javascript, "javascript");
compile_grammar(tree_sitter_examples::golang, "golang");
compile_grammar(tree_sitter_examples::arithmetic, "arithmetic");
compile_grammar(tree_sitter_examples::json, "json");
compile_grammar(tree_sitter_examples::javascript, "javascript");
compile_grammar(tree_sitter_examples::golang, "golang");
});
END_TEST

2
spec/compiler/compiler_specs.cc
View file

@ -1,5 +1,5 @@
#include "compiler/compiler_spec_helper.h"
int main(int argc, char *argv[]) {
return bandit::run(argc, argv);
return bandit::run(argc, argv);
}

48
spec/compiler/helpers/containers.h
View file

@ -17,35 +17,35 @@ using tree_sitter::rules::rule_ptr;
template<typename K>
class rule_map : public map<K, rule_ptr> {
public:
bool operator==(const map<K, rule_ptr> &other) const {
if (this->size() != other.size()) return false;
for (const auto &pair : *this) {
auto other_pair = other.find(pair.first);
if (other_pair == other.end()) return false;
if (!pair.second->operator==(*other_pair->second)) return false;
}
return true;
}
public:
bool operator==(const map<K, rule_ptr> &other) const {
if (this->size() != other.size()) return false;
for (const auto &pair : *this) {
auto other_pair = other.find(pair.first);
if (other_pair == other.end()) return false;
if (!pair.second->operator==(*other_pair->second)) return false;
}
return true;
}
rule_map(const initializer_list<pair<const K, rule_ptr>> &list) : map<K, rule_ptr>(list) {}
rule_map(const initializer_list<pair<const K, rule_ptr>> &list) : map<K, rule_ptr>(list) {}
};
class rule_list : public vector<pair<string, rule_ptr>> {
public:
bool operator==(const vector<pair<string, rule_ptr>> &other) const {
if (this->size() != other.size()) return false;
for (size_t i = 0; i < this->size(); i++) {
auto pair = this->operator[](i);
auto other_pair = other[i];
if (!pair.second->operator==(*other_pair.second))
return false;
}
return true;
}
public:
bool operator==(const vector<pair<string, rule_ptr>> &other) const {
if (this->size() != other.size()) return false;
for (size_t i = 0; i < this->size(); i++) {
auto pair = this->operator[](i);
auto other_pair = other[i];
if (!pair.second->operator==(*other_pair.second))
return false;
}
return true;
}
rule_list(const initializer_list<pair<string, rule_ptr>> &list) :
vector<pair<string, rule_ptr>>(list) {}
rule_list(const initializer_list<pair<string, rule_ptr>> &list) :
vector<pair<string, rule_ptr>>(list) {}
};

51
spec/compiler/helpers/equals_pointer.h
View file

@ -1,38 +1,37 @@
#ifndef HELPERS_EQUALS_POINTER_H_
#define HELPERS_EQUALS_POINTER_H_
#include "bandit/bandit.h"
#include <string>
namespace snowhouse {
using namespace std;
using namespace std;
template<typename ExpectedType>
struct EqualsPointerConstraint : Expression<EqualsPointerConstraint<ExpectedType>>
{
EqualsPointerConstraint(const ExpectedType& expected) : expected(expected) {}
template<typename ExpectedType>
struct EqualsPointerConstraint : Expression<EqualsPointerConstraint<ExpectedType>> {
EqualsPointerConstraint(const ExpectedType& expected) : expected(expected) {}
template<typename ActualType>
bool operator()(const ActualType& actual) const {
return *expected == *actual;
}
ExpectedType expected;
};
template<typename ExpectedType>
struct Stringizer<EqualsPointerConstraint<ExpectedType>>
{
static string ToString(const EqualsPointerConstraint<ExpectedType>& constraint) {
ostringstream builder;
builder << "pointer to " << snowhouse::Stringize(constraint.expected);
return builder.str();
}
};
template<typename ExpectedType>
inline EqualsPointerConstraint<ExpectedType> EqualsPointer(const ExpectedType& expected) {
return EqualsPointerConstraint<ExpectedType>(expected);
template<typename ActualType>
bool operator()(const ActualType& actual) const {
return *expected == *actual;
}
ExpectedType expected;
};
template<typename ExpectedType>
struct Stringizer<EqualsPointerConstraint<ExpectedType>> {
static string ToString(const EqualsPointerConstraint<ExpectedType>& constraint) {
ostringstream builder;
builder << "pointer to " << snowhouse::Stringize(constraint.expected);
return builder.str();
}
};
template<typename ExpectedType>
inline EqualsPointerConstraint<ExpectedType> EqualsPointer(const ExpectedType& expected) {
return EqualsPointerConstraint<ExpectedType>(expected);
}
}
#endif // HELPERS_EQUALS_POINTER_H_

68
spec/compiler/helpers/rule_helpers.cc
View file

@ -3,40 +3,40 @@
#include "compiler/rules/symbol.h"
namespace tree_sitter {
using std::make_shared;
using std::set;
using std::map;
using std::make_shared;
using std::set;
using std::map;
namespace rules {
rule_ptr character(const set<CharacterRange> &ranges) {
return make_shared<CharacterSet>(ranges);
}
rule_ptr character(const set<CharacterRange> &ranges, bool sign) {
if (sign)
return character(ranges);
else
return CharacterSet(ranges).complement().copy();
}
rule_ptr i_sym(size_t index) {
return make_shared<rules::Symbol>(index);
}
rule_ptr i_aux_sym(size_t index) {
return make_shared<rules::Symbol>(index, SymbolOptionAuxiliary);
}
rule_ptr i_token(size_t index) {
return make_shared<rules::Symbol>(index, SymbolOptionToken);
}
rule_ptr i_aux_token(size_t index) {
return make_shared<rules::Symbol>(index, SymbolOption(SymbolOptionAuxiliary|SymbolOptionToken));
}
rule_ptr metadata(rule_ptr rule, map<MetadataKey, int> values) {
return make_shared<Metadata>(rule, values);
}
namespace rules {
rule_ptr character(const set<CharacterRange> &ranges) {
return make_shared<CharacterSet>(ranges);
}
rule_ptr character(const set<CharacterRange> &ranges, bool sign) {
if (sign)
return character(ranges);
else
return CharacterSet(ranges).complement().copy();
}
rule_ptr i_sym(size_t index) {
return make_shared<rules::Symbol>(index);
}
rule_ptr i_aux_sym(size_t index) {
return make_shared<rules::Symbol>(index, SymbolOptionAuxiliary);
}
rule_ptr i_token(size_t index) {
return make_shared<rules::Symbol>(index, SymbolOptionToken);
}
rule_ptr i_aux_token(size_t index) {
return make_shared<rules::Symbol>(index, SymbolOption(SymbolOptionAuxiliary|SymbolOptionToken));
}
rule_ptr metadata(rule_ptr rule, map<MetadataKey, int> values) {
return make_shared<Metadata>(rule, values);
}
}
}

18
spec/compiler/helpers/rule_helpers.h
View file

@ -6,15 +6,15 @@
#include "compiler/rules/metadata.h"
namespace tree_sitter {
namespace rules {
rule_ptr metadata(rule_ptr, std::map<MetadataKey, int>);
rule_ptr character(const std::set<CharacterRange> &ranges);
rule_ptr character(const std::set<CharacterRange> &ranges, bool sign);
rule_ptr i_sym(size_t index);
rule_ptr i_aux_sym(size_t index);
rule_ptr i_token(size_t index);
rule_ptr i_aux_token(size_t index);
}
namespace rules {
rule_ptr metadata(rule_ptr, std::map<MetadataKey, int>);
rule_ptr character(const std::set<CharacterRange> &ranges);
rule_ptr character(const std::set<CharacterRange> &ranges, bool sign);
rule_ptr i_sym(size_t index);
rule_ptr i_aux_sym(size_t index);
rule_ptr i_token(size_t index);
rule_ptr i_aux_token(size_t index);
}
}
#endif

137
spec/compiler/helpers/stream_methods.h
View file

@ -12,74 +12,75 @@ using std::cout;
namespace std {
template<typename T>
inline std::ostream& operator<<(std::ostream &stream, const std::vector<T> &vector) {
stream << std::string("#<vector: ");
bool started = false;
for (auto item : vector) {
if (started) stream << std::string(", ");
stream << item;
started = true;
}
return stream << ">";
}
template<typename T>
inline std::ostream& operator<<(std::ostream &stream, const std::set<T> &set) {
stream << std::string("#<set: ");
bool started = false;
for (auto item : set) {
if (started) stream << std::string(", ");
stream << item;
started = true;
}
return stream << ">";
}
template<typename T>
inline std::ostream& operator<<(std::ostream &stream, const std::unordered_set<T> &set) {
stream << std::string("#<set: ");
bool started = false;
for (auto item : set) {
if (started) stream << std::string(", ");
stream << item;
started = true;
}
return stream << ">";
}
template<typename TKey, typename TValue>
inline std::ostream& operator<<(std::ostream &stream, const std::map<TKey, TValue> &map) {
stream << std::string("#<map: ");
bool started = false;
for (auto pair : map) {
if (started) stream << std::string(", ");
stream << pair.first;
stream << std::string(" => ");
stream << pair.second;
started = true;
}
return stream << ">";
}
template<typename TKey, typename TValue>
inline std::ostream& operator<<(std::ostream &stream, const std::unordered_map<TKey, TValue> &map) {
stream << std::string("#<map: ");
bool started = false;
for (auto pair : map) {
if (started) stream << std::string(", ");
stream << pair.first;
stream << std::string(" => ");
stream << pair.second;
started = true;
}
return stream << ">";
}
template<typename T1, typename T2>
inline std::ostream& operator<<(std::ostream &stream, const std::pair<T1, T2> &pair) {
return stream << "{" << pair.first << ", " << pair.second << "}";
}
template<typename T>
inline std::ostream& operator<<(std::ostream &stream, const std::vector<T> &vector) {
stream << std::string("#<vector: ");
bool started = false;
for (auto item : vector) {
if (started) stream << std::string(", ");
stream << item;
started = true;
}
return stream << ">";
}
template<typename T>
inline std::ostream& operator<<(std::ostream &stream, const std::set<T> &set) {
stream << std::string("#<set: ");
bool started = false;
for (auto item : set) {
if (started) stream << std::string(", ");
stream << item;
started = true;
}
return stream << ">";
}
template<typename T>
inline std::ostream& operator<<(std::ostream &stream, const std::unordered_set<T> &set) {
stream << std::string("#<set: ");
bool started = false;
for (auto item : set) {
if (started) stream << std::string(", ");
stream << item;
started = true;
}
return stream << ">";
}
template<typename TKey, typename TValue>
inline std::ostream& operator<<(std::ostream &stream, const std::map<TKey, TValue> &map) {
stream << std::string("#<map: ");
bool started = false;
for (auto pair : map) {
if (started) stream << std::string(", ");
stream << pair.first;
stream << std::string(" => ");
stream << pair.second;
started = true;
}
return stream << ">";
}
template<typename TKey, typename TValue>
inline std::ostream& operator<<(std::ostream &stream, const std::unordered_map<TKey, TValue> &map) {
stream << std::string("#<map: ");
bool started = false;
for (auto pair : map) {
if (started) stream << std::string(", ");
stream << pair.first;
stream << std::string(" => ");
stream << pair.second;
started = true;
}
return stream << ">";
}
template<typename T1, typename T2>
inline std::ostream& operator<<(std::ostream &stream, const std::pair<T1, T2> &pair) {
return stream << "{" << pair.first << ", " << pair.second << "}";
}
} // namespace std
#endif

162
spec/compiler/prepare_grammar/expand_repeats_spec.cc
View file

@ -9,105 +9,105 @@ using namespace rules;
using prepare_grammar::expand_repeats;
describe("expanding repeat rules in a grammar", []() {
it("replaces repeat rules with pairs of recursive rules", [&]() {
SyntaxGrammar grammar({
{ "rule0", repeat(i_token(0)) },
}, {}, set<Symbol>());
it("replaces repeat rules with pairs of recursive rules", [&]() {
SyntaxGrammar grammar({
{ "rule0", repeat(i_token(0)) },
}, {}, set<Symbol>());
auto match = expand_repeats(grammar);
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", i_aux_sym(0) },
})));
AssertThat(match.rules, Equals(rule_list({
{ "rule0", i_aux_sym(0) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({ seq({ i_token(0), i_aux_sym(0) }), blank() }) },
})));
});
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({ seq({ i_token(0), i_aux_sym(0) }), blank() }) },
})));
});
it("replaces repeats inside of sequences", [&]() {
SyntaxGrammar grammar({
{ "rule0", seq({ i_token(10), repeat(i_token(11)) }) },
}, {}, set<Symbol>());
it("replaces repeats inside of sequences", [&]() {
SyntaxGrammar grammar({
{ "rule0", seq({ i_token(10), repeat(i_token(11)) }) },
}, {}, set<Symbol>());
auto match = expand_repeats(grammar);
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", seq({ i_token(10), i_aux_sym(0) }) },
})));
AssertThat(match.rules, Equals(rule_list({
{ "rule0", seq({ i_token(10), i_aux_sym(0) }) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({ i_token(11), i_aux_sym(0) }),
blank() }) },
})));
});
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({ i_token(11), i_aux_sym(0) }),
blank() }) },
})));
});
it("replaces repeats inside of choices", [&]() {
SyntaxGrammar grammar({
{ "rule0", choice({ i_token(10), repeat(i_token(11)) }) },
}, {}, set<Symbol>());
it("replaces repeats inside of choices", [&]() {
SyntaxGrammar grammar({
{ "rule0", choice({ i_token(10), repeat(i_token(11)) }) },
}, {}, set<Symbol>());
auto match = expand_repeats(grammar);
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", choice({ i_token(10), i_aux_sym(0) }) },
})));
AssertThat(match.rules, Equals(rule_list({
{ "rule0", choice({ i_token(10), i_aux_sym(0) }) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({ i_token(11), i_aux_sym(0) }),
blank() }) },
})));
});
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({ i_token(11), i_aux_sym(0) }),
blank() }) },
})));
});
it("can replace multiple repeats in the same rule", [&]() {
SyntaxGrammar grammar({
{ "rule0", seq({ repeat(i_token(10)), repeat(i_token(11)) }) },
}, {}, set<Symbol>());
it("can replace multiple repeats in the same rule", [&]() {
SyntaxGrammar grammar({
{ "rule0", seq({ repeat(i_token(10)), repeat(i_token(11)) }) },
}, {}, set<Symbol>());
auto match = expand_repeats(grammar);
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", seq({ i_aux_sym(0), i_aux_sym(1) }) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({
i_token(10),
i_aux_sym(0) }),
blank() }) },
{ "rule0_repeat1", choice({
seq({
i_token(11),
i_aux_sym(1) }),
blank() }) },
})));
});
AssertThat(match.rules, Equals(rule_list({
{ "rule0", seq({ i_aux_sym(0), i_aux_sym(1) }) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({
i_token(10),
i_aux_sym(0) }),
blank() }) },
{ "rule0_repeat1", choice({
seq({
i_token(11),
i_aux_sym(1) }),
blank() }) },
})));
});
it("can replace repeats in multiple rules", [&]() {
SyntaxGrammar grammar({
{ "rule0", repeat(i_token(10)) },
{ "rule1", repeat(i_token(11)) },
}, {}, set<Symbol>());
it("can replace repeats in multiple rules", [&]() {
SyntaxGrammar grammar({
{ "rule0", repeat(i_token(10)) },
{ "rule1", repeat(i_token(11)) },
}, {}, set<Symbol>());
auto match = expand_repeats(grammar);
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", i_aux_sym(0) },
{ "rule1", i_aux_sym(1) },
})));
AssertThat(match.rules, Equals(rule_list({
{ "rule0", i_aux_sym(0) },
{ "rule1", i_aux_sym(1) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({ i_token(10), i_aux_sym(0) }),
blank() }) },
{ "rule1_repeat0", choice({
seq({ i_token(11), i_aux_sym(1) }),
blank() }) },
})));
});
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({ i_token(10), i_aux_sym(0) }),
blank() }) },
{ "rule1_repeat0", choice({
seq({ i_token(11), i_aux_sym(1) }),
blank() }) },
})));
});
});
END_TEST

84
spec/compiler/prepare_grammar/expand_tokens_spec.cc
View file

@ -9,56 +9,56 @@ using namespace rules;
using prepare_grammar::expand_tokens;
describe("expanding token rules", []() {
it("replaces regex patterns with their expansion", [&]() {
LexicalGrammar grammar({
{ "rule_A", seq({
i_sym(10),
pattern("x*"),
i_sym(11) }) },
}, {});
it("replaces regex patterns with their expansion", [&]() {
LexicalGrammar grammar({
{ "rule_A", seq({
i_sym(10),
pattern("x*"),
i_sym(11) }) },
}, {});
auto result = expand_tokens(grammar);
auto result = expand_tokens(grammar);
AssertThat(result.second, Equals((const GrammarError *)nullptr));
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({
i_sym(10),
repeat(character({ 'x' })),
i_sym(11) }) },
})));
});
AssertThat(result.second, Equals((const GrammarError *)nullptr));
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({
i_sym(10),
repeat(character({ 'x' })),
i_sym(11) }) },
})));
});
it("replaces string rules with a sequence of characters", [&]() {
LexicalGrammar grammar({
{ "rule_A", seq({
i_sym(10),
str("xyz"),
i_sym(11) }) },
}, {});
it("replaces string rules with a sequence of characters", [&]() {
LexicalGrammar grammar({
{ "rule_A", seq({
i_sym(10),
str("xyz"),
i_sym(11) }) },
}, {});
auto result = expand_tokens(grammar);
auto result = expand_tokens(grammar);
AssertThat(result.second, Equals((const GrammarError *)nullptr));
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({
i_sym(10),
seq({ character({ 'x' }), character({ 'y' }), character({ 'z' }) }),
i_sym(11) }) },
})));
});
AssertThat(result.second, Equals((const GrammarError *)nullptr));
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({
i_sym(10),
seq({ character({ 'x' }), character({ 'y' }), character({ 'z' }) }),
i_sym(11) }) },
})));
});
it("returns an error when the grammar contains an invalid regex", [&]() {
LexicalGrammar grammar({
{ "rule_A", seq({
pattern("("),
str("xyz"),
pattern("[") }) },
}, {});
it("returns an error when the grammar contains an invalid regex", [&]() {
LexicalGrammar grammar({
{ "rule_A", seq({
pattern("("),
str("xyz"),
pattern("[") }) },
}, {});
auto result = expand_tokens(grammar);
auto result = expand_tokens(grammar);
AssertThat(result.second, EqualsPointer(new GrammarError(GrammarErrorTypeRegex, "unmatched open paren")));
});
AssertThat(result.second, EqualsPointer(new GrammarError(GrammarErrorTypeRegex, "unmatched open paren")));
});
});
END_TEST

294
spec/compiler/prepare_grammar/extract_tokens_spec.cc
View file

@ -12,174 +12,174 @@ using prepare_grammar::extract_tokens;
using prepare_grammar::InternedGrammar;
describe("extracting tokens from a grammar", []() {
it("moves string rules into the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", seq({ str("ab"), i_sym(0) }) }
},
set<Symbol>(),
set<char>()
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0) }) }
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, Equals(rule_list({
{ "'ab'", str("ab") },
})));
it("moves string rules into the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", seq({ str("ab"), i_sym(0) }) }
},
set<Symbol>(),
set<char>()
});
it("moves pattern rules into the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", seq({ pattern("a+"), i_sym(0) }) }
},
set<Symbol>(),
set<char>()
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0) }) }
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, Equals(rule_list({
{ "'ab'", str("ab") },
})));
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0) }) }
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, Equals(rule_list({
{ "/a+/", pattern("a+") },
})));
it("moves pattern rules into the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", seq({ pattern("a+"), i_sym(0) }) }
},
set<Symbol>(),
set<char>()
});
it("moves other rules marked as tokens into the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", seq({
token(seq({ pattern("."), choice({ str("a"), str("b") }) })),
i_sym(0) }) }
},
set<Symbol>(),
set<char>()
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0) }) }
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, Equals(rule_list({
{ "/a+/", pattern("a+") },
})));
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0) }) }
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, Equals(rule_list({
{ "(seq /./ (choice 'a' 'b'))", token(seq({ pattern("."), choice({ str("a"), str("b") }) })) },
})));
it("moves other rules marked as tokens into the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", seq({
token(seq({ pattern("."), choice({ str("a"), str("b") }) })),
i_sym(0) }) }
},
set<Symbol>(),
set<char>()
});
it("does not extract blanks", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", choice({ i_sym(0), blank() }) },
},
set<Symbol>(),
set<char>()
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0) }) }
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, Equals(rule_list({
{ "(seq /./ (choice 'a' 'b'))", token(seq({ pattern("."), choice({ str("a"), str("b") }) })) },
})));
});
AssertThat(result.first.rules, Equals(rule_list({
it("does not extract blanks", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", choice({ i_sym(0), blank() }) },
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, IsEmpty())
},
set<Symbol>(),
set<char>()
});
it("does not create duplicate tokens in the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", seq({ str("ab"), i_sym(0), str("ab") }) },
},
set<Symbol>(),
set<char>()
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", choice({ i_sym(0), blank() }) },
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, IsEmpty())
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0), i_aux_token(0) }) }
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, Equals(rule_list({
{ "'ab'", str("ab") },
})))
it("does not create duplicate tokens in the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", seq({ str("ab"), i_sym(0), str("ab") }) },
},
set<Symbol>(),
set<char>()
});
it("preserves the separator characters in the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", str("ab") },
},
set<Symbol>(),
{ 'x', 'y', 'z' }
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", seq({ i_aux_token(0), i_sym(0), i_aux_token(0) }) }
})));
AssertThat(result.first.aux_rules, IsEmpty())
AssertThat(result.second.rules, IsEmpty())
AssertThat(result.second.aux_rules, Equals(rule_list({
{ "'ab'", str("ab") },
})))
});
AssertThat(result.second.separators, Equals(set<char>({ 'x', 'y', 'z' })));
it("preserves the separator characters in the lexical grammar", [&]() {
pair<SyntaxGrammar, LexicalGrammar> result = extract_tokens(InternedGrammar{
{
{ "rule_A", str("ab") },
},
set<Symbol>(),
{ 'x', 'y', 'z' }
});
describe("when an entire rule can be extracted", [&]() {
it("moves the rule the lexical grammar when possible and updates referencing symbols", [&]() {
auto result = extract_tokens(InternedGrammar{
{
{ "rule_A", i_sym(1) },
{ "rule_B", pattern("a|b") },
{ "rule_C", token(seq({ str("a"), str("b") })) },
},
set<Symbol>(),
set<char>()
});
AssertThat(result.second.separators, Equals(set<char>({ 'x', 'y', 'z' })));
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", i_token(0) }
})));
AssertThat(result.first.aux_rules, IsEmpty());
AssertThat(result.second.rules, Equals(rule_list({
{ "rule_B", pattern("a|b") },
{ "rule_C", token(seq({ str("a"), str("b") })) },
})));
AssertThat(result.second.aux_rules, IsEmpty());
});
describe("when an entire rule can be extracted", [&]() {
it("moves the rule the lexical grammar when possible and updates referencing symbols", [&]() {
auto result = extract_tokens(InternedGrammar{
{
{ "rule_A", i_sym(1) },
{ "rule_B", pattern("a|b") },
{ "rule_C", token(seq({ str("a"), str("b") })) },
},
set<Symbol>(),
set<char>()
});
it("updates symbols whose indices need to change due to deleted rules", [&]() {
auto result = extract_tokens(InternedGrammar{
{
{ "rule_A", str("ab") },
{ "rule_B", i_sym(0) },
{ "rule_C", i_sym(1) },
},
set<Symbol>(),
set<char>()
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_B", i_token(0) },
{ "rule_C", i_sym(0) },
})));
AssertThat(result.first.aux_rules, IsEmpty());
AssertThat(result.second.rules, Equals(rule_list({
{ "rule_A", str("ab") },
})));
AssertThat(result.second.aux_rules, IsEmpty());
});
it("updates the grammar's ubiquitous_tokens", [&]() {
auto result = extract_tokens(InternedGrammar{
{
{ "rule_A", str("ab") },
{ "rule_B", i_sym(0) },
{ "rule_C", i_sym(1) },
},
{ Symbol(0) },
set<char>()
});
AssertThat(result.first.ubiquitous_tokens, Equals(set<Symbol>({
{ Symbol(0, SymbolOptionToken) }
})));
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_A", i_token(0) }
})));
AssertThat(result.first.aux_rules, IsEmpty());
AssertThat(result.second.rules, Equals(rule_list({
{ "rule_B", pattern("a|b") },
{ "rule_C", token(seq({ str("a"), str("b") })) },
})));
AssertThat(result.second.aux_rules, IsEmpty());
});
it("updates symbols whose indices need to change due to deleted rules", [&]() {
auto result = extract_tokens(InternedGrammar{
{
{ "rule_A", str("ab") },
{ "rule_B", i_sym(0) },
{ "rule_C", i_sym(1) },
},
set<Symbol>(),
set<char>()
});
AssertThat(result.first.rules, Equals(rule_list({
{ "rule_B", i_token(0) },
{ "rule_C", i_sym(0) },
})));
AssertThat(result.first.aux_rules, IsEmpty());
AssertThat(result.second.rules, Equals(rule_list({
{ "rule_A", str("ab") },
})));
AssertThat(result.second.aux_rules, IsEmpty());
});
it("updates the grammar's ubiquitous_tokens", [&]() {
auto result = extract_tokens(InternedGrammar{
{
{ "rule_A", str("ab") },
{ "rule_B", i_sym(0) },
{ "rule_C", i_sym(1) },
},
{ Symbol(0) },
set<char>()
});
AssertThat(result.first.ubiquitous_tokens, Equals(set<Symbol>({
{ Symbol(0, SymbolOptionToken) }
})));
});
});
});
END_TEST

84
spec/compiler/prepare_grammar/intern_symbols_spec.cc
View file

@ -11,59 +11,59 @@ using namespace rules;
using prepare_grammar::intern_symbols;
describe("interning symbols in a grammar", []() {
it("replaces named symbols with numerically-indexed symbols", [&]() {
Grammar grammar({
{ "x", choice({ sym("y"), sym("z") }) },
{ "y", sym("z") },
{ "z", str("stuff") }
});
auto result = intern_symbols(grammar);
AssertThat(result.second, Equals((GrammarError *)nullptr));
AssertThat(result.first.rules, Equals(rule_list({
{ "x", choice({ i_sym(1), i_sym(2) }) },
{ "y", i_sym(2) },
{ "z", str("stuff") },
})));
it("replaces named symbols with numerically-indexed symbols", [&]() {
Grammar grammar({
{ "x", choice({ sym("y"), sym("z") }) },
{ "y", sym("z") },
{ "z", str("stuff") }
});
describe("when there are symbols that reference undefined rules", [&]() {
it("returns an error", []() {
Grammar grammar({
{ "x", sym("y") },
});
auto result = intern_symbols(grammar);
auto result = intern_symbols(grammar);
AssertThat(result.second, Equals((GrammarError *)nullptr));
AssertThat(result.first.rules, Equals(rule_list({
{ "x", choice({ i_sym(1), i_sym(2) }) },
{ "y", i_sym(2) },
{ "z", str("stuff") },
})));
});
AssertThat(result.second->message, Equals("Undefined rule 'y'"));
});
describe("when there are symbols that reference undefined rules", [&]() {
it("returns an error", []() {
Grammar grammar({
{ "x", sym("y") },
});
auto result = intern_symbols(grammar);
AssertThat(result.second->message, Equals("Undefined rule 'y'"));
});
});
it("translates the grammar's optional 'ubiquitous_tokens' to numerical symbols", [&]() {
auto grammar = Grammar({
{ "x", choice({ sym("y"), sym("z") }) },
{ "y", sym("z") },
{ "z", str("stuff") }
}).ubiquitous_tokens({ "z" });
it("translates the grammar's optional 'ubiquitous_tokens' to numerical symbols", [&]() {
auto grammar = Grammar({
{ "x", choice({ sym("y"), sym("z") }) },
{ "y", sym("z") },
{ "z", str("stuff") }
}).ubiquitous_tokens({ "z" });
auto result = intern_symbols(grammar);
auto result = intern_symbols(grammar);
AssertThat(result.second, Equals((GrammarError *)nullptr));
AssertThat(result.first.ubiquitous_tokens, Equals(set<Symbol>({
Symbol(2)
})));
});
AssertThat(result.second, Equals((GrammarError *)nullptr));
AssertThat(result.first.ubiquitous_tokens, Equals(set<Symbol>({
Symbol(2)
})));
});
it("preserves the grammar's separator character set", [&]() {
auto grammar = Grammar({
{ "z", str("stuff") }
}).separators({ 'x', 'y' });
it("preserves the grammar's separator character set", [&]() {
auto grammar = Grammar({
{ "z", str("stuff") }
}).separators({ 'x', 'y' });
auto result = intern_symbols(grammar);
auto result = intern_symbols(grammar);
AssertThat(result.first.separators, Equals(set<char>({ 'x', 'y' })))
});
AssertThat(result.first.separators, Equals(set<char>({ 'x', 'y' })))
});
});
END_TEST

398
spec/compiler/prepare_grammar/parse_regex_spec.cc
View file

@ -7,225 +7,225 @@ using namespace rules;
using prepare_grammar::parse_regex;
describe("parsing regex patterns", []() {
struct ValidInputRow {
string description;
string pattern;
rule_ptr rule;
};
struct ValidInputRow {
string description;
string pattern;
rule_ptr rule;
};
vector<ValidInputRow> valid_inputs = {
{
"character sets",
"[aAeE]",
character({ 'a', 'A', 'e', 'E' })
},
vector<ValidInputRow> valid_inputs = {
{
"character sets",
"[aAeE]",
character({ 'a', 'A', 'e', 'E' })
},
{
"'.' characters as wildcards",
".",
CharacterSet({'\n'}).complement().copy()
},
{
"'.' characters as wildcards",
".",
CharacterSet({'\n'}).complement().copy()
},
{
"character classes",
"\\w-\\d",
seq({
character({ {'a', 'z'}, {'A', 'Z'}, {'0', '9'} }),
character({ '-' }),
character({ {'0', '9'} }) })
},
{
"character classes",
"\\w-\\d",
seq({
character({ {'a', 'z'}, {'A', 'Z'}, {'0', '9'} }),
character({ '-' }),
character({ {'0', '9'} }) })
},
{
"choices",
"ab|cd|ef",
choice({
seq({
character({ 'a' }),
character({ 'b' }),
}),
seq({
character({ 'c' }),
character({ 'd' })
}),
seq({
character({ 'e' }),
character({ 'f' })
})
})
},
{
"choices",
"ab|cd|ef",
choice({
seq({
character({ 'a' }),
character({ 'b' }),
}),
seq({
character({ 'c' }),
character({ 'd' })
}),
seq({
character({ 'e' }),
character({ 'f' })
})
})
},
{
"simple sequences",
"abc",
seq({
character({ 'a' }),
character({ 'b' }),
character({ 'c' }) })
},
{
"simple sequences",
"abc",
seq({
character({ 'a' }),
character({ 'b' }),
character({ 'c' }) })
},
{
"character ranges",
"[12a-dA-D3]",
character({ {'1', '3'}, {'a', 'd'}, { 'A', 'D' }, })
},
{
"character ranges",
"[12a-dA-D3]",
character({ {'1', '3'}, {'a', 'd'}, { 'A', 'D' }, })
},
{
"negated characters",
"[^a\\d]",
character({ {'a'}, {'0', '9'} }, false)
},
{
"negated characters",
"[^a\\d]",
character({ {'a'}, {'0', '9'} }, false)
},
{
"backslashes",
"\\\\",
character({ '\\' })
},
{
"backslashes",
"\\\\",
character({ '\\' })
},
{
"character groups in sequences",
"x([^x]|\\\\x)*x",
seq({
character({ 'x' }),
repeat(choice({
character({ 'x' }, false),
seq({ character({ '\\' }), character({ 'x' }) })
})),
character({ 'x' })
})
},
{
"character groups in sequences",
"x([^x]|\\\\x)*x",
seq({
character({ 'x' }),
repeat(choice({
character({ 'x' }, false),
seq({ character({ '\\' }), character({ 'x' }) })
})),
character({ 'x' })
})
},
{
"choices in sequences",
"(a|b)cd",
seq({
choice({
character({ 'a' }),
character({ 'b' }),
}),
character({ 'c' }),
character({ 'd' })
})
},
{
"choices in sequences",
"(a|b)cd",
seq({
choice({
character({ 'a' }),
character({ 'b' }),
}),
character({ 'c' }),
character({ 'd' })
})
},
{
"escaped parentheses",
"a\\(b",
seq({
character({ 'a' }),
character({ '(' }),
character({ 'b' })
})
},
{
"escaped parentheses",
"a\\(b",
seq({
character({ 'a' }),
character({ '(' }),
character({ 'b' })
})
},
{
"escaped periods",
"a\\.",
seq({
character({ 'a' }),
character({ '.' })
})
},
{
"escaped periods",
"a\\.",
seq({
character({ 'a' }),
character({ '.' })
})
},
{
"escaped characters",
"\\t\\n\\r",
seq({
character({ '\t' }),
character({ '\n' }),
character({ '\r' }),
})
},
{
"escaped characters",
"\\t\\n\\r",
seq({
character({ '\t' }),
character({ '\n' }),
character({ '\r' }),
})
},
{
"plus repeats",
"(ab)+(cd)+",
seq({
seq({
seq({ character({ 'a' }), character({ 'b' }) }),
repeat(seq({ character({ 'a' }), character({ 'b' }) })),
}),
seq({
seq({ character({ 'c' }), character({ 'd' }) }),
repeat(seq({ character({ 'c' }), character({ 'd' }) })),
}),
})
},
{
"plus repeats",
"(ab)+(cd)+",
seq({
seq({
seq({ character({ 'a' }), character({ 'b' }) }),
repeat(seq({ character({ 'a' }), character({ 'b' }) })),
}),
seq({
seq({ character({ 'c' }), character({ 'd' }) }),
repeat(seq({ character({ 'c' }), character({ 'd' }) })),
}),
})
},
{
"asterix repeats",
"(ab)*(cd)*",
seq({
repeat(seq({ character({ 'a' }), character({ 'b' }) })),
repeat(seq({ character({ 'c' }), character({ 'd' }) })),
})
},
{
"asterix repeats",
"(ab)*(cd)*",
seq({
repeat(seq({ character({ 'a' }), character({ 'b' }) })),
repeat(seq({ character({ 'c' }), character({ 'd' }) })),
})
},
{
"optional rules",
"a(bc)?",
seq({
character({ 'a' }),
choice({
seq({ character({ 'b' }), character({ 'c' }) }),
blank()
})
})
}
};
{
"optional rules",
"a(bc)?",
seq({
character({ 'a' }),
choice({
seq({ character({ 'b' }), character({ 'c' }) }),
blank()
})
})
}
};
struct InvalidInputRow {
string description;
string pattern;
const char *message;
};
struct InvalidInputRow {
string description;
string pattern;
const char *message;
};
vector<InvalidInputRow> invalid_inputs = {
{
"mismatched open parens",
"(a",
"unmatched open paren",
},
{
"mismatched nested open parens",
"((a) (b)",
"unmatched open paren",
},
{
"mismatched close parens",
"a)",
"unmatched close paren",
},
{
"mismatched nested close parens",
"((a) b))",
"unmatched close paren",
},
{
"mismatched brackets for character classes",
"[a",
"unmatched open square bracket",
},
{
"mismatched brackets for character classes",
"a]",
"unmatched close square bracket",
},
};
vector<InvalidInputRow> invalid_inputs = {
{
"mismatched open parens",
"(a",
"unmatched open paren",
},
{
"mismatched nested open parens",
"((a) (b)",
"unmatched open paren",
},
{
"mismatched close parens",
"a)",
"unmatched close paren",
},
{
"mismatched nested close parens",
"((a) b))",
"unmatched close paren",
},
{
"mismatched brackets for character classes",
"[a",
"unmatched open square bracket",
},
{
"mismatched brackets for character classes",
"a]",
"unmatched close square bracket",
},
};
for (auto &row : valid_inputs) {
it(("parses " + row.description).c_str(), [&]() {
auto result = parse_regex(row.pattern);
AssertThat(result.first, EqualsPointer(row.rule));
});
}
for (auto &row : valid_inputs) {
it(("parses " + row.description).c_str(), [&]() {
auto result = parse_regex(row.pattern);
AssertThat(result.first, EqualsPointer(row.rule));
});
}
for (auto &row : invalid_inputs) {
it(("handles invalid regexes with " + row.description).c_str(), [&]() {
auto result = parse_regex(row.pattern);
AssertThat(result.second, !Equals((const GrammarError *)nullptr));
AssertThat(result.second->message, Contains(row.message));
});
}
for (auto &row : invalid_inputs) {
it(("handles invalid regexes with " + row.description).c_str(), [&]() {
auto result = parse_regex(row.pattern);
AssertThat(result.second, !Equals((const GrammarError *)nullptr));
AssertThat(result.second->message, Contains(row.message));
});
}
});
END_TEST

188
spec/compiler/rules/character_set_spec.cc
View file

@ -6,107 +6,107 @@ using namespace rules;
START_TEST
describe("character sets", []() {
unsigned char max_char = 255;
unsigned char max_char = 255;
describe("computing the complement", [&]() {
it("works for the set containing only the null character", [&]() {
CharacterSet set1({ '\0' });
auto set2 = set1.complement();
AssertThat(set2, Equals(CharacterSet({
{ 1, max_char }
})));
AssertThat(set2.complement(), Equals(set1));
});
it("works for single character sets", [&]() {
CharacterSet set1({ 'b' });
auto set2 = set1.complement();
AssertThat(set2, Equals(CharacterSet({
{ 0, 'a' },
{ 'c', max_char },
})));
AssertThat(set2.complement(), Equals(set1));
});
describe("computing the complement", [&]() {
it("works for the set containing only the null character", [&]() {
CharacterSet set1({ '\0' });
auto set2 = set1.complement();
AssertThat(set2, Equals(CharacterSet({
{ 1, max_char }
})));
AssertThat(set2.complement(), Equals(set1));
});
describe("computing unions", [&]() {
it("works for disjoint sets", [&]() {
CharacterSet set({ {'a', 'z'} });
set.add_set(CharacterSet({ {'A', 'Z'} }));
AssertThat(set, Equals(CharacterSet({ {'a', 'z'}, {'A', 'Z'} })));
});
it("works for single character sets", [&]() {
CharacterSet set1({ 'b' });
auto set2 = set1.complement();
AssertThat(set2, Equals(CharacterSet({
{ 0, 'a' },
{ 'c', max_char },
})));
AssertThat(set2.complement(), Equals(set1));
});
});
it("works for sets with adjacent ranges", [&]() {
CharacterSet set({ CharacterRange('a', 'r') });
set.add_set(CharacterSet({ CharacterRange('s', 'z') }));
AssertThat(set, Equals(CharacterSet({ {'a', 'z'} })));
});
it("becomes the complete set when the complement is added", [&]() {
CharacterSet set({ 'c' });
auto complement = set.complement();
set.add_set(complement);
AssertThat(set, Equals(CharacterSet({ {0, max_char} })));
});
it("works when the result becomes a continuous range", []() {
CharacterSet set({ {'a', 'd'}, {'f', 'z'} });
set.add_set(CharacterSet({ {'c', 'g'} }));
AssertThat(set, Equals(CharacterSet({ {'a', 'z'} })));
});
it("does nothing for the set of all characters", [&]() {
CharacterSet set({ 'a' });
set.add_set(set.complement());
AssertThat(set, Equals(CharacterSet({ {'\0', max_char} })));
});
describe("computing unions", [&]() {
it("works for disjoint sets", [&]() {
CharacterSet set({ {'a', 'z'} });
set.add_set(CharacterSet({ {'A', 'Z'} }));
AssertThat(set, Equals(CharacterSet({ {'a', 'z'}, {'A', 'Z'} })));
});
describe("subtracting sets", []() {
CharacterSet intersection;
it("works for disjoint sets", [&]() {
CharacterSet set1({ {'a', 'z'} });
intersection = set1.remove_set(CharacterSet({ {'A', 'Z'} }));
AssertThat(set1, Equals(CharacterSet({ {'a', 'z'} })));
AssertThat(intersection, Equals(CharacterSet()));
});
it("works when one set is a proper subset of the other", [&]() {
CharacterSet set1({ {'a','z'} });
intersection = set1.remove_set(CharacterSet({ {'d', 's'} }));
AssertThat(set1, Equals(CharacterSet({ {'a', 'c'}, {'t', 'z'} })));
AssertThat(intersection, Equals(CharacterSet({ {'d', 's'} })));
});
it("works for a set that overlaps the right side", [&]() {
CharacterSet set1({ {'a','s'} });
intersection = set1.remove_set(CharacterSet({ {'m', 'z'} }));
AssertThat(set1, Equals(CharacterSet({ {'a', 'l'} })));
AssertThat(intersection, Equals(CharacterSet({ {'m', 's'} })));
});
it("works for a set that overlaps the left side", [&]() {
CharacterSet set2({ {'m','z'} });
intersection = set2.remove_set(CharacterSet({ {'a', 's'} }));
AssertThat(set2, Equals(CharacterSet({ {'t', 'z'} })));
AssertThat(intersection, Equals(CharacterSet({ {'m', 's'} })));
});
it("works for sets with multiple ranges", [&]() {
CharacterSet set1({ {'a', 'd'}, {'m', 'z'} });
intersection = set1.remove_set(CharacterSet({ {'c', 'o'}, {'s', 'x'} }));
AssertThat(set1, Equals(CharacterSet({ {'a', 'b'}, {'p', 'r'}, {'y', 'z'} })));
AssertThat(intersection, Equals(CharacterSet({ {'c', 'd'}, {'m', 'o'}, {'s', 'x'} })));
});
it("works when the result is empty", [&]() {
CharacterSet set1({ 'd' });
intersection = set1.remove_set(CharacterSet({ 'a', 'd', 'x' }));
AssertThat(set1, Equals(CharacterSet()));
AssertThat(intersection, Equals(CharacterSet({ 'd' })));
});
it("works for sets with adjacent ranges", [&]() {
CharacterSet set({ CharacterRange('a', 'r') });
set.add_set(CharacterSet({ CharacterRange('s', 'z') }));
AssertThat(set, Equals(CharacterSet({ {'a', 'z'} })));
});
it("becomes the complete set when the complement is added", [&]() {
CharacterSet set({ 'c' });
auto complement = set.complement();
set.add_set(complement);
AssertThat(set, Equals(CharacterSet({ {0, max_char} })));
});
it("works when the result becomes a continuous range", []() {
CharacterSet set({ {'a', 'd'}, {'f', 'z'} });
set.add_set(CharacterSet({ {'c', 'g'} }));
AssertThat(set, Equals(CharacterSet({ {'a', 'z'} })));
});
it("does nothing for the set of all characters", [&]() {
CharacterSet set({ 'a' });
set.add_set(set.complement());
AssertThat(set, Equals(CharacterSet({ {'\0', max_char} })));
});
});
describe("subtracting sets", []() {
CharacterSet intersection;
it("works for disjoint sets", [&]() {
CharacterSet set1({ {'a', 'z'} });
intersection = set1.remove_set(CharacterSet({ {'A', 'Z'} }));
AssertThat(set1, Equals(CharacterSet({ {'a', 'z'} })));
AssertThat(intersection, Equals(CharacterSet()));
});
it("works when one set is a proper subset of the other", [&]() {
CharacterSet set1({ {'a','z'} });
intersection = set1.remove_set(CharacterSet({ {'d', 's'} }));
AssertThat(set1, Equals(CharacterSet({ {'a', 'c'}, {'t', 'z'} })));
AssertThat(intersection, Equals(CharacterSet({ {'d', 's'} })));
});
it("works for a set that overlaps the right side", [&]() {
CharacterSet set1({ {'a','s'} });
intersection = set1.remove_set(CharacterSet({ {'m', 'z'} }));
AssertThat(set1, Equals(CharacterSet({ {'a', 'l'} })));
AssertThat(intersection, Equals(CharacterSet({ {'m', 's'} })));
});
it("works for a set that overlaps the left side", [&]() {
CharacterSet set2({ {'m','z'} });
intersection = set2.remove_set(CharacterSet({ {'a', 's'} }));
AssertThat(set2, Equals(CharacterSet({ {'t', 'z'} })));
AssertThat(intersection, Equals(CharacterSet({ {'m', 's'} })));
});
it("works for sets with multiple ranges", [&]() {
CharacterSet set1({ {'a', 'd'}, {'m', 'z'} });
intersection = set1.remove_set(CharacterSet({ {'c', 'o'}, {'s', 'x'} }));
AssertThat(set1, Equals(CharacterSet({ {'a', 'b'}, {'p', 'r'}, {'y', 'z'} })));
AssertThat(intersection, Equals(CharacterSet({ {'c', 'd'}, {'m', 'o'}, {'s', 'x'} })));
});
it("works when the result is empty", [&]() {
CharacterSet set1({ 'd' });
intersection = set1.remove_set(CharacterSet({ 'a', 'd', 'x' }));
AssertThat(set1, Equals(CharacterSet()));
AssertThat(intersection, Equals(CharacterSet({ 'd' })));
});
});
});
END_TEST

96
spec/runtime/helpers/read_test_entries.cc
View file

@ -30,46 +30,46 @@ using std::ifstream;
using std::istreambuf_iterator;
static string trim_output(const string &input) {
string result(input);
result = regex_replace(result, regex("[\n\t ]+", extended), string(" "));
result = regex_replace(result, regex("^ ", extended), string(""));
result = regex_replace(result, regex(" $", extended), string(""));
result = regex_replace(result, regex("\\) \\)", extended), string("))"));
return result;
string result(input);
result = regex_replace(result, regex("[\n\t ]+", extended), string(" "));
result = regex_replace(result, regex("^ ", extended), string(""));
result = regex_replace(result, regex(" $", extended), string(""));
result = regex_replace(result, regex("\\) \\)", extended), string("))"));
return result;
}
static vector<TestEntry> get_test_entries_from_string(string content) {
regex header_pattern("===+\n" "([^=]+)\n" "===+", extended);
regex separator_pattern("---+", extended);
vector<string> descriptions;
vector<string> bodies;
regex header_pattern("===+\n" "([^=]+)\n" "===+", extended);
regex separator_pattern("---+", extended);
vector<string> descriptions;
vector<string> bodies;
for (;;) {
smatch matches;
if (!regex_search(content, matches, header_pattern) || matches.empty())
break;
for (;;) {
smatch matches;
if (!regex_search(content, matches, header_pattern) || matches.empty())
break;
string description = matches[1].str();
descriptions.push_back(description);
string description = matches[1].str();
descriptions.push_back(description);
if (!bodies.empty())
bodies.back().erase(matches.position());
content.erase(0, matches.position() + matches[0].length());
bodies.push_back(content);
}
if (!bodies.empty())
bodies.back().erase(matches.position());
content.erase(0, matches.position() + matches[0].length());
bodies.push_back(content);
}
vector<TestEntry> result;
for (size_t i = 0; i < descriptions.size(); i++) {
string body = bodies[i];
smatch matches;
if (regex_search(body, matches, separator_pattern)) {
result.push_back({
descriptions[i],
body.substr(0, matches.position()),
trim_output(body.substr(matches.position() + matches[0].length()))
});
}
for (size_t i = 0; i < descriptions.size(); i++) {
string body = bodies[i];
smatch matches;
if (regex_search(body, matches, separator_pattern)) {
result.push_back({
descriptions[i],
body.substr(0, matches.position()),
trim_output(body.substr(matches.position() + matches[0].length()))
});
}
}
return result;
}
@ -79,9 +79,9 @@ static vector<string> list_directory(string dir_name) {
DIR *dir = opendir(dir_name.c_str());
if (!dir) {
printf("\nTest error - no such directory '%s'", dir_name.c_str());
return result;
}
printf("\nTest error - no such directory '%s'", dir_name.c_str());
return result;
}
struct dirent *dir_entry;
while ((dir_entry = readdir(dir))) {
@ -90,27 +90,27 @@ static vector<string> list_directory(string dir_name) {
result.push_back(dir_name + "/" + name);
}
closedir(dir);
closedir(dir);
return result;
}
static string src_dir() {
const char *dir = getenv("TREESITTER_DIR");
if (!dir) dir = getenv("PWD");
return dir;
const char *dir = getenv("TREESITTER_DIR");
if (!dir) dir = getenv("PWD");
return dir;
}
vector<TestEntry> test_entries_for_language(string language) {
vector<TestEntry> result;
string language_dir = src_dir() + "/spec/runtime/languages/" + language;
vector<string> filenames = list_directory(language_dir);
vector<TestEntry> result;
string language_dir = src_dir() + "/spec/runtime/languages/" + language;
vector<string> filenames = list_directory(language_dir);
for (string &filename : filenames) {
ifstream file(filename);
std::string content((istreambuf_iterator<char>(file)), istreambuf_iterator<char>());
for (TestEntry &entry : get_test_entries_from_string(content))
result.push_back(entry);
}
for (string &filename : filenames) {
ifstream file(filename);
std::string content((istreambuf_iterator<char>(file)), istreambuf_iterator<char>());
for (TestEntry &entry : get_test_entries_from_string(content))
result.push_back(entry);
}
return result;
return result;
}

2
spec/runtime/helpers/read_test_entries.h
View file

@ -5,7 +5,7 @@
#include <vector>
struct TestEntry {
std::string description;
std::string description;
std::string input;
std::string tree_string;
};

24
spec/runtime/helpers/spy_reader.cc
View file

@ -4,21 +4,21 @@
using std::string;
static const char * spy_read(void *data, size_t *bytes_read) {
SpyReader *reader = static_cast<SpyReader *>(data);
size_t size = std::min(reader->chunk_size,
reader->content.length() - reader->position);
const char *result = reader->content.data() + reader->position;
reader->strings_read.back() += string(result, size);
reader->position += size;
*bytes_read = size;
return result;
SpyReader *reader = static_cast<SpyReader *>(data);
size_t size = std::min(reader->chunk_size,
reader->content.length() - reader->position);
const char *result = reader->content.data() + reader->position;
reader->strings_read.back() += string(result, size);
reader->position += size;
*bytes_read = size;
return result;
}
static int spy_seek(void *data, size_t position) {
SpyReader *reader = static_cast<SpyReader *>(data);
reader->strings_read.push_back("");
reader->position = position;
return 0;
SpyReader *reader = static_cast<SpyReader *>(data);
reader->strings_read.push_back("");
reader->position = position;
return 0;
}
SpyReader::SpyReader(string content, size_t chunk_size) :

14
spec/runtime/helpers/spy_reader.h
View file

@ -6,14 +6,14 @@
#include "tree_sitter/runtime.h"
class SpyReader {
public:
SpyReader(std::string content, size_t chunk_size);
public:
SpyReader(std::string content, size_t chunk_size);
std::string content;
size_t position;
size_t chunk_size;
TSInput input;
std::vector<std::string> strings_read;
std::string content;
size_t position;
size_t chunk_size;
TSInput input;
std::vector<std::string> strings_read;
};
#endif // HELPERS_SPY_READER_H_

8
spec/runtime/helpers/tree_helpers.cc
View file

@ -1,8 +1,8 @@
#include "runtime/helpers/tree_helpers.h"
TSTree ** tree_array(std::vector<TSTree *> trees) {
TSTree ** result = (TSTree **)calloc(trees.size(), sizeof(TSTree *));
for (size_t i = 0; i < trees.size(); i++)
result[i] = trees[i];
return result;
TSTree ** result = (TSTree **)calloc(trees.size(), sizeof(TSTree *));
for (size_t i = 0; i < trees.size(); i++)
result[i] = trees[i];
return result;
}

50
spec/runtime/languages/language_specs.cc
View file

@ -9,37 +9,37 @@ extern "C" const TSLanguage *ts_language_golang();
START_TEST
describe("Languages", [&]() {
TSDocument *doc;
TSDocument *doc;
before_each([&]() {
doc = ts_document_make();
});
before_each([&]() {
doc = ts_document_make();
});
after_each([&]() {
ts_document_free(doc);
});
after_each([&]() {
ts_document_free(doc);
});
auto run_tests_for_language = [&](string language_name, const TSLanguage *language) {
describe(language_name.c_str(), [&]() {
before_each([&]() {
ts_document_set_language(doc, language);
});
auto run_tests_for_language = [&](string language_name, const TSLanguage *language) {
describe(language_name.c_str(), [&]() {
before_each([&]() {
ts_document_set_language(doc, language);
});
for (auto &entry : test_entries_for_language(language_name)) {
it(entry.description.c_str(), [&]() {
ts_document_set_input_string(doc, entry.input.c_str());
auto doc_string = ts_document_string(doc);
AssertThat(doc_string, Equals(entry.tree_string.c_str()));
free((void *)doc_string);
});
}
for (auto &entry : test_entries_for_language(language_name)) {
it(entry.description.c_str(), [&]() {
ts_document_set_input_string(doc, entry.input.c_str());
auto doc_string = ts_document_string(doc);
AssertThat(doc_string, Equals(entry.tree_string.c_str()));
free((void *)doc_string);
});
};
}
});
};
run_tests_for_language("json", ts_language_json());
run_tests_for_language("arithmetic", ts_language_arithmetic());
run_tests_for_language("javascript", ts_language_javascript());
run_tests_for_language("golang", ts_language_golang());
run_tests_for_language("json", ts_language_json());
run_tests_for_language("arithmetic", ts_language_arithmetic());
run_tests_for_language("javascript", ts_language_javascript());
run_tests_for_language("golang", ts_language_golang());
});
END_TEST

186
spec/runtime/node_spec.cc
View file

@ -5,104 +5,104 @@ extern "C" TSLanguage * ts_language_json();
START_TEST
describe("Node", []() {
TSDocument *document;
TSNode *root;
TSDocument *document;
TSNode *root;
before_each([&]() {
document = ts_document_make();
ts_document_set_language(document, ts_language_json());
before_each([&]() {
document = ts_document_make();
ts_document_set_language(document, ts_language_json());
ts_document_set_input_string(document, " [12, 5, 345]");
root = ts_document_root_node(document);
AssertThat(ts_node_string(root), Equals("(array (number) (number) (number))"));
ts_document_set_input_string(document, " [12, 5, 345]");
root = ts_document_root_node(document);
AssertThat(ts_node_string(root), Equals("(array (number) (number) (number))"));
});
after_each([&]() {
ts_document_free(document);
ts_node_release(root);
});
describe("child_count", [&]() {
it("returns the number of visible child nodes", [&]() {
AssertThat(ts_node_child_count(root), Equals<size_t>(3));
});
});
describe("child(i)", [&]() {
it("returns the child node at the given index", [&]() {
TSNode *number1 = ts_node_child(root, 0);
TSNode *number2 = ts_node_child(root, 1);
TSNode *number3 = ts_node_child(root, 2);
AssertThat(ts_node_name(root), Equals("array"));
AssertThat(ts_node_name(number1), Equals("number"));
AssertThat(ts_node_name(number2), Equals("number"));
AssertThat(ts_node_name(number3), Equals("number"));
AssertThat(ts_node_pos(root), Equals<size_t>(2));
AssertThat(ts_node_size(root), Equals<size_t>(12));
AssertThat(ts_node_pos(number1), Equals<size_t>(3));
AssertThat(ts_node_size(number1), Equals<size_t>(2));
AssertThat(ts_node_pos(number2), Equals<size_t>(7));
AssertThat(ts_node_size(number2), Equals<size_t>(1));
AssertThat(ts_node_pos(number3), Equals<size_t>(10));
AssertThat(ts_node_size(number3), Equals<size_t>(3));
ts_node_release(number1);
ts_node_release(number2);
ts_node_release(number3);
});
});
describe("parent", [&]() {
it("returns the node's parent node", [&]() {
TSNode *number = ts_node_child(root, 1);
AssertThat(ts_node_parent(number), Equals(root));
ts_node_release(number);
});
});
describe("next_sibling and prev_sibling", [&]() {
it("returns the node's next and previous siblings", [&]() {
TSNode *number1 = ts_node_child(root, 0);
TSNode *number2 = ts_node_child(root, 1);
TSNode *number3 = ts_node_child(root, 2);
AssertThat(ts_node_eq(ts_node_next_sibling(number2), number3), IsTrue());
AssertThat(ts_node_eq(ts_node_prev_sibling(number2), number1), IsTrue());
ts_node_release(number1);
ts_node_release(number2);
ts_node_release(number3);
});
});
describe("leaf_at_pos", [&]() {
it("can retrieve the leaf node at a given position", [&]() {
TSNode *number1 = ts_node_leaf_at_pos(root, 3);
TSNode *number2 = ts_node_leaf_at_pos(root, 7);
AssertThat(ts_node_name(number1), Equals("number"));
AssertThat(ts_node_size(number1), Equals<size_t>(2));
AssertThat(ts_node_name(number2), Equals("number"));
AssertThat(ts_node_size(number2), Equals<size_t>(1));
ts_node_release(number1);
ts_node_release(number2);
});
after_each([&]() {
ts_document_free(document);
ts_node_release(root);
});
describe("child_count", [&]() {
it("returns the number of visible child nodes", [&]() {
AssertThat(ts_node_child_count(root), Equals<size_t>(3));
});
});
describe("child(i)", [&]() {
it("returns the child node at the given index", [&]() {
TSNode *number1 = ts_node_child(root, 0);
TSNode *number2 = ts_node_child(root, 1);
TSNode *number3 = ts_node_child(root, 2);
AssertThat(ts_node_name(root), Equals("array"));
AssertThat(ts_node_name(number1), Equals("number"));
AssertThat(ts_node_name(number2), Equals("number"));
AssertThat(ts_node_name(number3), Equals("number"));
AssertThat(ts_node_pos(root), Equals<size_t>(2));
AssertThat(ts_node_size(root), Equals<size_t>(12));
AssertThat(ts_node_pos(number1), Equals<size_t>(3));
AssertThat(ts_node_size(number1), Equals<size_t>(2));
AssertThat(ts_node_pos(number2), Equals<size_t>(7));
AssertThat(ts_node_size(number2), Equals<size_t>(1));
AssertThat(ts_node_pos(number3), Equals<size_t>(10));
AssertThat(ts_node_size(number3), Equals<size_t>(3));
ts_node_release(number1);
ts_node_release(number2);
ts_node_release(number3);
});
});
describe("parent", [&]() {
it("returns the node's parent node", [&]() {
TSNode *number = ts_node_child(root, 1);
AssertThat(ts_node_parent(number), Equals(root));
ts_node_release(number);
});
});
describe("next_sibling and prev_sibling", [&]() {
it("returns the node's next and previous siblings", [&]() {
TSNode *number1 = ts_node_child(root, 0);
TSNode *number2 = ts_node_child(root, 1);
TSNode *number3 = ts_node_child(root, 2);
AssertThat(ts_node_eq(ts_node_next_sibling(number2), number3), IsTrue());
AssertThat(ts_node_eq(ts_node_prev_sibling(number2), number1), IsTrue());
ts_node_release(number1);
ts_node_release(number2);
ts_node_release(number3);
});
});
describe("leaf_at_pos", [&]() {
it("can retrieve the leaf node at a given position", [&]() {
TSNode *number1 = ts_node_leaf_at_pos(root, 3);
TSNode *number2 = ts_node_leaf_at_pos(root, 7);
AssertThat(ts_node_name(number1), Equals("number"));
AssertThat(ts_node_size(number1), Equals<size_t>(2));
AssertThat(ts_node_name(number2), Equals("number"));
AssertThat(ts_node_size(number2), Equals<size_t>(1));
ts_node_release(number1);
ts_node_release(number2);
});
it("returns higher-level nodes when no leaf is at the given position", [&]() {
TSNode *node = ts_node_leaf_at_pos(root, 6);
AssertThat(ts_node_name(node), Equals("array"));
ts_node_release(node);
});
it("returns higher-level nodes when no leaf is at the given position", [&]() {
TSNode *node = ts_node_leaf_at_pos(root, 6);
AssertThat(ts_node_name(node), Equals("array"));
ts_node_release(node);
});
});
});
END_TEST

98
spec/runtime/parser_spec.cc
View file

@ -10,69 +10,69 @@ TSStateId lex_fn_state_received;
TSLexer *lex_fn_lexer_received;
TSTree * fake_lex(TSLexer *lexer, TSStateId state_id) {
lex_fn_lexer_received = lexer;
lex_fn_state_received = state_id;
return lex_fn_node_to_return;
lex_fn_lexer_received = lexer;
lex_fn_state_received = state_id;
return lex_fn_node_to_return;
}
START_TEST
describe("LR Parsers", [&]() {
TSParser parser;
SpyReader *reader;
TSLanguage language;
TSParser parser;
SpyReader *reader;
TSLanguage language;
before_each([&]() {
language = *dummy_language();
language.lex_fn = fake_lex;
parser = ts_parser_make(&language);
reader = new SpyReader("some structured text", 5);
});
after_each([&]() {
ts_parser_destroy(&parser);
delete reader;
});
describe("when starting at the beginning of the input (edit is NULL)", [&]() {
before_each([&]() {
language = *dummy_language();
language.lex_fn = fake_lex;
parser = ts_parser_make(&language);
reader = new SpyReader("some structured text", 5);
ts_parser_start(&parser, reader->input, nullptr);
});
after_each([&]() {
ts_parser_destroy(&parser);
delete reader;
it("runs the lexer with the lex state corresponding to the initial state", [&]() {
lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym2, 5, 1, 0);
ts_parser_step(&parser);
AssertThat(lex_fn_state_received, Equals(100));
});
describe("when starting at the beginning of the input (edit is NULL)", [&]() {
before_each([&]() {
ts_parser_start(&parser, reader->input, nullptr);
});
describe("when the returned symbol indicates a shift action", [&]() {
before_each([&]() {
lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym2, 5, 1, 0);
});
it("runs the lexer with the lex state corresponding to the initial state", [&]() {
lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym2, 5, 1, 0);
ts_parser_step(&parser);
AssertThat(lex_fn_state_received, Equals(100));
});
it("advances to the state specified in the action", [&]() {
ts_parser_step(&parser);
AssertThat(ts_stack_top_state(&parser.stack), Equals(12));
});
describe("when the returned symbol indicates a shift action", [&]() {
before_each([&]() {
lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym2, 5, 1, 0);
});
it("advances to the state specified in the action", [&]() {
ts_parser_step(&parser);
AssertThat(ts_stack_top_state(&parser.stack), Equals(12));
});
it("continues parsing (returns NULL)", [&]() {
auto result = ts_parser_step(&parser);
AssertThat(result, Equals((TSTree *)nullptr));
});
});
describe("when the returned symbol indicates an error", [&]() {
before_each([&]() {
lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym1, 5, 1, 0);
});
it("ends the parse, returning an error tree", [&]() {
auto result = ts_parser_step(&parser);
AssertThat(result->symbol, Equals(ts_builtin_sym_error));
});
});
it("continues parsing (returns NULL)", [&]() {
auto result = ts_parser_step(&parser);
AssertThat(result, Equals((TSTree *)nullptr));
});
});
describe("when the returned symbol indicates an error", [&]() {
before_each([&]() {
lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym1, 5, 1, 0);
});
it("ends the parse, returning an error tree", [&]() {
auto result = ts_parser_step(&parser);
AssertThat(result->symbol, Equals(ts_builtin_sym_error));
});
});
});
});
END_TEST

2
spec/runtime/runtime_specs.cc
View file

@ -1,5 +1,5 @@
#include "runtime/runtime_spec_helper.h"
int main(int argc, char *argv[]) {
return bandit::run(argc, argv);
return bandit::run(argc, argv);
}

142
spec/runtime/stack_spec.cc
View file

@ -8,89 +8,89 @@ enum { sym1, sym2, hidden_sym };
int hidden_symbols[] = { 0, 0, 1 };
describe("stacks", [&]() {
TSStack stack;
TSStack stack;
before_each([&]() {
stack = ts_stack_make();
});
after_each([&]() {
ts_stack_delete(&stack);
});
it("starts out empty", [&]() {
AssertThat(stack.size, Equals<size_t>(0));
AssertThat(ts_stack_top_state(&stack), Equals(0));
AssertThat(ts_stack_top_node(&stack), Equals((TSTree *)nullptr));
});
describe("pushing a symbol", [&]() {
TSTree *node1;
before_each([&]() {
stack = ts_stack_make();
node1 = ts_tree_make_leaf(sym1, 5, 1, 0);
ts_stack_push(&stack, 5, node1);
});
after_each([&]() {
ts_stack_delete(&stack);
ts_tree_release(node1);
});
it("starts out empty", [&]() {
AssertThat(stack.size, Equals<size_t>(0));
AssertThat(ts_stack_top_state(&stack), Equals(0));
AssertThat(ts_stack_top_node(&stack), Equals((TSTree *)nullptr));
it("adds the symbol to the stack", [&]() {
AssertThat(stack.size, Equals<size_t>(1));
AssertThat(ts_stack_top_state(&stack), Equals(5));
AssertThat(ts_stack_top_node(&stack), Equals(node1));
});
});
describe("reducing a symbol", [&]() {
TSTree **nodes;
before_each([&]() {
nodes = tree_array({
ts_tree_make_leaf(sym1, 5, 1, 0),
ts_tree_make_leaf(sym1, 5, 1, 0),
ts_tree_make_leaf(hidden_sym, 5, 1, 0),
ts_tree_make_leaf(sym1, 5, 1, 0),
});
for (TSStateId i = 0; i < 4; i++)
ts_stack_push(&stack, 10 + i, nodes[i]);
});
describe("pushing a symbol", [&]() {
TSTree *node1;
before_each([&]() {
node1 = ts_tree_make_leaf(sym1, 5, 1, 0);
ts_stack_push(&stack, 5, node1);
});
after_each([&]() {
ts_tree_release(node1);
});
it("adds the symbol to the stack", [&]() {
AssertThat(stack.size, Equals<size_t>(1));
AssertThat(ts_stack_top_state(&stack), Equals(5));
AssertThat(ts_stack_top_node(&stack), Equals(node1));
});
after_each([&]() {
for (TSStateId i = 0; i < 4; i++)
ts_tree_release(nodes[i]);
free(nodes);
});
describe("reducing a symbol", [&]() {
TSTree **nodes;
before_each([&]() {
nodes = tree_array({
ts_tree_make_leaf(sym1, 5, 1, 0),
ts_tree_make_leaf(sym1, 5, 1, 0),
ts_tree_make_leaf(hidden_sym, 5, 1, 0),
ts_tree_make_leaf(sym1, 5, 1, 0),
});
for (TSStateId i = 0; i < 4; i++)
ts_stack_push(&stack, 10 + i, nodes[i]);
});
after_each([&]() {
for (TSStateId i = 0; i < 4; i++)
ts_tree_release(nodes[i]);
free(nodes);
});
it("pops the given number of nodes off the stack", [&]() {
AssertThat(stack.size, Equals<size_t>(4));
ts_stack_reduce(&stack, sym2, 3, hidden_symbols, 0);
AssertThat(stack.size, Equals<size_t>(1));
});
it("returns a node with the given symbol", [&]() {
TSTree *node = ts_stack_reduce(&stack, sym2, 3, hidden_symbols, 0);
AssertThat(node->symbol, Equals(sym2));
});
it("removes any hidden nodes from its regular list of children", [&]() {
TSTree *expected_children[3] = {
stack.entries[1].node,
stack.entries[2].node,
stack.entries[3].node,
};
TSTree *node = ts_stack_reduce(&stack, sym2, 3, hidden_symbols, 0);
size_t child_count;
TSTree **children = ts_tree_children(node, &child_count);
AssertThat(child_count, Equals<size_t>(3));
for (size_t i = 0; i < 2; i++)
AssertThat(children[i], Equals(expected_children[i]));
});
it("pops the given number of nodes off the stack", [&]() {
AssertThat(stack.size, Equals<size_t>(4));
ts_stack_reduce(&stack, sym2, 3, hidden_symbols, 0);
AssertThat(stack.size, Equals<size_t>(1));
});
it("returns a node with the given symbol", [&]() {
TSTree *node = ts_stack_reduce(&stack, sym2, 3, hidden_symbols, 0);
AssertThat(node->symbol, Equals(sym2));
});
it("removes any hidden nodes from its regular list of children", [&]() {
TSTree *expected_children[3] = {
stack.entries[1].node,
stack.entries[2].node,
stack.entries[3].node,
};
TSTree *node = ts_stack_reduce(&stack, sym2, 3, hidden_symbols, 0);
size_t child_count;
TSTree **children = ts_tree_children(node, &child_count);
AssertThat(child_count, Equals<size_t>(3));
for (size_t i = 0; i < 2; i++)
AssertThat(children[i], Equals(expected_children[i]));
});
});
});
END_TEST

122
spec/runtime/tree_spec.cc
View file

@ -12,80 +12,80 @@ enum {
static const char *names[] = { "error", "end", "cat", "dog", "pig" };
describe("trees", []() {
TSTree *tree1, *tree2, *parent1;
TSTree *tree1, *tree2, *parent1;
before_each([&]() {
tree1 = ts_tree_make_leaf(cat, 5, 2, 0);
tree2 = ts_tree_make_leaf(cat, 3, 1, 0);
parent1 = ts_tree_make_node(dog, 2, tree_array({
tree1, tree2, // children
}), 0);
before_each([&]() {
tree1 = ts_tree_make_leaf(cat, 5, 2, 0);
tree2 = ts_tree_make_leaf(cat, 3, 1, 0);
parent1 = ts_tree_make_node(dog, 2, tree_array({
tree1, tree2, // children
}), 0);
});
after_each([&]() {
ts_tree_release(tree1);
ts_tree_release(tree2);
ts_tree_release(parent1);
});
describe("making a parent node", [&]() {
it("computes its size based on its child nodes", [&]() {
AssertThat(parent1->size, Equals<size_t>(9));
});
after_each([&]() {
ts_tree_release(tree1);
ts_tree_release(tree2);
ts_tree_release(parent1);
it("computes its offset based on its first child", [&]() {
AssertThat(parent1->offset, Equals<size_t>(2));
});
});
describe("equality", [&]() {
it("returns true for identical trees", [&]() {
TSTree *tree1_copy = ts_tree_make_leaf(cat, 5, 2, 0);
AssertThat(ts_tree_equals(tree1, tree1_copy), Equals(1));
TSTree *tree2_copy = ts_tree_make_leaf(cat, 3, 1, 0);
AssertThat(ts_tree_equals(tree2, tree2_copy), Equals(1));
TSTree *parent2 = ts_tree_make_node(dog, 2, tree_array({
tree1_copy, tree2_copy,
}), 0);
AssertThat(ts_tree_equals(parent1, parent2), Equals(1));
ts_tree_release(tree1_copy);
ts_tree_release(tree2_copy);
ts_tree_release(parent2);
});
describe("making a parent node", [&]() {
it("computes its size based on its child nodes", [&]() {
AssertThat(parent1->size, Equals<size_t>(9));
});
it("computes its offset based on its first child", [&]() {
AssertThat(parent1->offset, Equals<size_t>(2));
});
it("returns false for trees with different symbols", [&]() {
TSTree *different_tree = ts_tree_make_leaf(pig, 0, 0, 0);
AssertThat(ts_tree_equals(tree1, different_tree), Equals(0));
ts_tree_release(different_tree);
});
describe("equality", [&]() {
it("returns true for identical trees", [&]() {
TSTree *tree1_copy = ts_tree_make_leaf(cat, 5, 2, 0);
AssertThat(ts_tree_equals(tree1, tree1_copy), Equals(1));
TSTree *tree2_copy = ts_tree_make_leaf(cat, 3, 1, 0);
AssertThat(ts_tree_equals(tree2, tree2_copy), Equals(1));
it("returns false for trees with different children", [&]() {
TSTree *different_tree = ts_tree_make_leaf(pig, 0, 0, 0);
TSTree *different_parent = ts_tree_make_node(dog, 2, tree_array({
different_tree, different_tree,
}), 0);
TSTree *parent2 = ts_tree_make_node(dog, 2, tree_array({
tree1_copy, tree2_copy,
}), 0);
AssertThat(ts_tree_equals(parent1, parent2), Equals(1));
AssertThat(ts_tree_equals(different_parent, parent1), Equals(0));
AssertThat(ts_tree_equals(parent1, different_parent), Equals(0));
ts_tree_release(tree1_copy);
ts_tree_release(tree2_copy);
ts_tree_release(parent2);
});
it("returns false for trees with different symbols", [&]() {
TSTree *different_tree = ts_tree_make_leaf(pig, 0, 0, 0);
AssertThat(ts_tree_equals(tree1, different_tree), Equals(0));
ts_tree_release(different_tree);
});
it("returns false for trees with different children", [&]() {
TSTree *different_tree = ts_tree_make_leaf(pig, 0, 0, 0);
TSTree *different_parent = ts_tree_make_node(dog, 2, tree_array({
different_tree, different_tree,
}), 0);
AssertThat(ts_tree_equals(different_parent, parent1), Equals(0));
AssertThat(ts_tree_equals(parent1, different_parent), Equals(0));
ts_tree_release(different_tree);
ts_tree_release(different_parent);
});
ts_tree_release(different_tree);
ts_tree_release(different_parent);
});
});
describe("serialization", [&]() {
it("returns a readable string", [&]() {
auto string1 = ts_tree_string(tree1, names);
AssertThat(string(string1), Equals("(cat)"));
free(string1);
describe("serialization", [&]() {
it("returns a readable string", [&]() {
auto string1 = ts_tree_string(tree1, names);
AssertThat(string(string1), Equals("(cat)"));
free(string1);
auto string2 = ts_tree_string(parent1, names);
AssertThat(string(string2), Equals("(dog (cat) (cat))"));
free(string2);
});
auto string2 = ts_tree_string(parent1, names);
AssertThat(string(string2), Equals("(dog (cat) (cat))"));
free(string2);
});
});
});
END_TEST