🎨
This commit is contained in:
Max Brunsfeld
parent
686dc0997c
commit
abf8a4f2c2
28 changed files with 313 additions and 356 deletions
|
|
@ -20,15 +20,18 @@ describe("recovery_tokens(rule)", []() {
|
|||
};
|
||||
|
||||
grammar.variables = {
|
||||
LexicalVariable("var0", VariableTypeNamed, character({}, false), false),
|
||||
LexicalVariable("var1", VariableTypeNamed, seq({
|
||||
LexicalVariable{"var0", VariableTypeNamed, character({}, false), false},
|
||||
LexicalVariable{"var1", VariableTypeNamed, seq({
|
||||
character({ 'a', 'b' }),
|
||||
character({}, false),
|
||||
character({ 'c', 'd' }),
|
||||
}), false),
|
||||
}), false},
|
||||
};
|
||||
|
||||
AssertThat(get_compatible_tokens(grammar).recovery_tokens, Equals<set<Symbol>>({ Symbol(1, Symbol::Terminal) }));
|
||||
AssertThat(
|
||||
get_compatible_tokens(grammar).recovery_tokens,
|
||||
Equals<set<Symbol>>({ Symbol(1, Symbol::Terminal) })
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -26,13 +26,13 @@ describe("ParseItemSetBuilder", []() {
|
|||
|
||||
it("adds items at the beginnings of referenced rules", [&]() {
|
||||
SyntaxGrammar grammar{{
|
||||
SyntaxVariable("rule0", VariableTypeNamed, {
|
||||
SyntaxVariable{"rule0", VariableTypeNamed, {
|
||||
Production({
|
||||
{Symbol(1, Symbol::NonTerminal), 0, AssociativityNone},
|
||||
{Symbol(11, Symbol::Terminal), 0, AssociativityNone},
|
||||
}),
|
||||
}),
|
||||
SyntaxVariable("rule1", VariableTypeNamed, {
|
||||
}},
|
||||
SyntaxVariable{"rule1", VariableTypeNamed, {
|
||||
Production({
|
||||
{Symbol(12, Symbol::Terminal), 0, AssociativityNone},
|
||||
{Symbol(13, Symbol::Terminal), 0, AssociativityNone},
|
||||
|
|
@ -40,13 +40,13 @@ describe("ParseItemSetBuilder", []() {
|
|||
Production({
|
||||
{Symbol(2, Symbol::NonTerminal), 0, AssociativityNone},
|
||||
})
|
||||
}),
|
||||
SyntaxVariable("rule2", VariableTypeNamed, {
|
||||
}},
|
||||
SyntaxVariable{"rule2", VariableTypeNamed, {
|
||||
Production({
|
||||
{Symbol(14, Symbol::Terminal), 0, AssociativityNone},
|
||||
{Symbol(15, Symbol::Terminal), 0, AssociativityNone},
|
||||
})
|
||||
}),
|
||||
}},
|
||||
}, {}, {}, {}};
|
||||
|
||||
auto production = [&](int variable_index, int production_index) -> const Production & {
|
||||
|
|
@ -85,19 +85,19 @@ describe("ParseItemSetBuilder", []() {
|
|||
|
||||
it("handles rules with empty productions", [&]() {
|
||||
SyntaxGrammar grammar{{
|
||||
SyntaxVariable("rule0", VariableTypeNamed, {
|
||||
SyntaxVariable{"rule0", VariableTypeNamed, {
|
||||
Production({
|
||||
{Symbol(1, Symbol::NonTerminal), 0, AssociativityNone},
|
||||
{Symbol(11, Symbol::Terminal), 0, AssociativityNone},
|
||||
}),
|
||||
}),
|
||||
SyntaxVariable("rule1", VariableTypeNamed, {
|
||||
}},
|
||||
SyntaxVariable{"rule1", VariableTypeNamed, {
|
||||
Production({
|
||||
{Symbol(12, Symbol::Terminal), 0, AssociativityNone},
|
||||
{Symbol(13, Symbol::Terminal), 0, AssociativityNone},
|
||||
}),
|
||||
Production({})
|
||||
}),
|
||||
}},
|
||||
}, {}, {}, {}};
|
||||
|
||||
auto production = [&](int variable_index, int production_index) -> const Production & {
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
#include "compiler/prepare_grammar/initial_syntax_grammar.h"
|
||||
#include "compiler/prepare_grammar/expand_repeats.h"
|
||||
#include "helpers/rule_helpers.h"
|
||||
#include "helpers/stream_methods.h"
|
||||
|
||||
START_TEST
|
||||
|
||||
|
|
@ -11,141 +12,159 @@ using prepare_grammar::expand_repeats;
|
|||
|
||||
describe("expand_repeats", []() {
|
||||
it("replaces repeat rules with pairs of recursive rules", [&]() {
|
||||
InitialSyntaxGrammar grammar{{
|
||||
Variable("rule0", VariableTypeNamed, repeat1(i_token(0))),
|
||||
}, {}, {}, {}};
|
||||
InitialSyntaxGrammar grammar{
|
||||
{
|
||||
Variable{"rule0", VariableTypeNamed, repeat1(i_token(0))},
|
||||
},
|
||||
{}, {}, {}
|
||||
};
|
||||
|
||||
auto result = expand_repeats(grammar);
|
||||
|
||||
AssertThat(result.variables, Equals(vector<Variable>({
|
||||
Variable("rule0", VariableTypeNamed, i_sym(1)),
|
||||
Variable("rule0_repeat1", VariableTypeAuxiliary, choice({
|
||||
AssertThat(result.variables, Equals(vector<Variable>{
|
||||
Variable{"rule0", VariableTypeNamed, i_sym(1)},
|
||||
Variable{"rule0_repeat1", VariableTypeAuxiliary, choice({
|
||||
seq({ i_sym(1), i_token(0) }),
|
||||
i_token(0),
|
||||
})),
|
||||
})));
|
||||
})},
|
||||
}));
|
||||
});
|
||||
|
||||
it("replaces repeats inside of sequences", [&]() {
|
||||
InitialSyntaxGrammar grammar{{
|
||||
Variable("rule0", VariableTypeNamed, seq({
|
||||
i_token(10),
|
||||
repeat1(i_token(11)),
|
||||
})),
|
||||
}, {}, {}, {}};
|
||||
InitialSyntaxGrammar grammar{
|
||||
{
|
||||
Variable{"rule0", VariableTypeNamed, seq({
|
||||
i_token(10),
|
||||
repeat1(i_token(11)),
|
||||
})},
|
||||
},
|
||||
{}, {}, {}
|
||||
};
|
||||
|
||||
auto result = expand_repeats(grammar);
|
||||
|
||||
AssertThat(result.variables, Equals(vector<Variable>({
|
||||
Variable("rule0", VariableTypeNamed, seq({
|
||||
AssertThat(result.variables, Equals(vector<Variable>{
|
||||
Variable{"rule0", VariableTypeNamed, seq({
|
||||
i_token(10),
|
||||
i_sym(1),
|
||||
})),
|
||||
Variable("rule0_repeat1", VariableTypeAuxiliary, choice({
|
||||
})},
|
||||
Variable{"rule0_repeat1", VariableTypeAuxiliary, choice({
|
||||
seq({ i_sym(1), i_token(11) }),
|
||||
i_token(11)
|
||||
})),
|
||||
})));
|
||||
})},
|
||||
}));
|
||||
});
|
||||
|
||||
it("replaces repeats inside of choices", [&]() {
|
||||
InitialSyntaxGrammar grammar{{
|
||||
Variable("rule0", VariableTypeNamed, choice({
|
||||
i_token(10),
|
||||
repeat1(i_token(11))
|
||||
})),
|
||||
}, {}, {}, {}};
|
||||
InitialSyntaxGrammar grammar{
|
||||
{
|
||||
Variable{"rule0", VariableTypeNamed, choice({
|
||||
i_token(10),
|
||||
repeat1(i_token(11))
|
||||
})},
|
||||
},
|
||||
{}, {}, {}
|
||||
};
|
||||
|
||||
auto result = expand_repeats(grammar);
|
||||
|
||||
AssertThat(result.variables, Equals(vector<Variable>({
|
||||
Variable("rule0", VariableTypeNamed, choice({
|
||||
AssertThat(result.variables, Equals(vector<Variable>{
|
||||
Variable{"rule0", VariableTypeNamed, choice({
|
||||
i_token(10),
|
||||
i_sym(1),
|
||||
})),
|
||||
Variable("rule0_repeat1", VariableTypeAuxiliary, choice({
|
||||
})},
|
||||
Variable{"rule0_repeat1", VariableTypeAuxiliary, choice({
|
||||
seq({ i_sym(1), i_token(11) }),
|
||||
i_token(11),
|
||||
})),
|
||||
})));
|
||||
})},
|
||||
}));
|
||||
});
|
||||
|
||||
it("does not create redundant auxiliary rules", [&]() {
|
||||
InitialSyntaxGrammar grammar{{
|
||||
Variable("rule0", VariableTypeNamed, choice({
|
||||
seq({ i_token(1), repeat1(i_token(4)) }),
|
||||
seq({ i_token(2), repeat1(i_token(4)) }),
|
||||
})),
|
||||
Variable("rule1", VariableTypeNamed, seq({
|
||||
i_token(3),
|
||||
repeat1(i_token(4))
|
||||
})),
|
||||
}, {}, {}, {}};
|
||||
InitialSyntaxGrammar grammar{
|
||||
{
|
||||
Variable{"rule0", VariableTypeNamed, choice({
|
||||
seq({ i_token(1), repeat1(i_token(4)) }),
|
||||
seq({ i_token(2), repeat1(i_token(4)) }),
|
||||
})},
|
||||
Variable{"rule1", VariableTypeNamed, seq({
|
||||
i_token(3),
|
||||
repeat1(i_token(4))
|
||||
})},
|
||||
},
|
||||
{}, {}, {}
|
||||
};
|
||||
|
||||
auto result = expand_repeats(grammar);
|
||||
|
||||
AssertThat(result.variables, Equals(vector<Variable>({
|
||||
Variable("rule0", VariableTypeNamed, choice({
|
||||
AssertThat(result.variables, Equals(vector<Variable>{
|
||||
Variable{"rule0", VariableTypeNamed, choice({
|
||||
seq({ i_token(1), i_sym(2) }),
|
||||
seq({ i_token(2), i_sym(2) }),
|
||||
})),
|
||||
Variable("rule1", VariableTypeNamed, seq({
|
||||
})},
|
||||
Variable{"rule1", VariableTypeNamed, seq({
|
||||
i_token(3),
|
||||
i_sym(2),
|
||||
})),
|
||||
Variable("rule0_repeat1", VariableTypeAuxiliary, choice({
|
||||
})},
|
||||
Variable{"rule0_repeat1", VariableTypeAuxiliary, choice({
|
||||
seq({ i_sym(2), i_token(4) }),
|
||||
i_token(4),
|
||||
})),
|
||||
})));
|
||||
})},
|
||||
}));
|
||||
});
|
||||
|
||||
it("can replace multiple repeats in the same rule", [&]() {
|
||||
InitialSyntaxGrammar grammar{{
|
||||
Variable("rule0", VariableTypeNamed, seq({
|
||||
repeat1(i_token(10)),
|
||||
repeat1(i_token(11)),
|
||||
})),
|
||||
}, {}, {}, {}};
|
||||
InitialSyntaxGrammar grammar{
|
||||
{
|
||||
Variable{"rule0", VariableTypeNamed, seq({
|
||||
repeat1(i_token(10)),
|
||||
repeat1(i_token(11)),
|
||||
})},
|
||||
},
|
||||
{}, {}, {}
|
||||
};
|
||||
|
||||
auto result = expand_repeats(grammar);
|
||||
|
||||
AssertThat(result.variables, Equals(vector<Variable>({
|
||||
Variable("rule0", VariableTypeNamed, seq({
|
||||
AssertThat(result.variables, Equals(vector<Variable>{
|
||||
Variable{"rule0", VariableTypeNamed, seq({
|
||||
i_sym(1),
|
||||
i_sym(2),
|
||||
})),
|
||||
Variable("rule0_repeat1", VariableTypeAuxiliary, choice({
|
||||
})},
|
||||
Variable{"rule0_repeat1", VariableTypeAuxiliary, choice({
|
||||
seq({ i_sym(1), i_token(10) }),
|
||||
i_token(10),
|
||||
})),
|
||||
Variable("rule0_repeat2", VariableTypeAuxiliary, choice({
|
||||
})},
|
||||
Variable{"rule0_repeat2", VariableTypeAuxiliary, choice({
|
||||
seq({ i_sym(2), i_token(11) }),
|
||||
i_token(11),
|
||||
})),
|
||||
})));
|
||||
})},
|
||||
}));
|
||||
});
|
||||
|
||||
it("can replace repeats in multiple rules", [&]() {
|
||||
InitialSyntaxGrammar grammar{{
|
||||
Variable("rule0", VariableTypeNamed, repeat1(i_token(10))),
|
||||
Variable("rule1", VariableTypeNamed, repeat1(i_token(11))),
|
||||
}, {}, {}, {}};
|
||||
InitialSyntaxGrammar grammar{
|
||||
{
|
||||
Variable{"rule0", VariableTypeNamed, repeat1(i_token(10))},
|
||||
Variable{"rule1", VariableTypeNamed, repeat1(i_token(11))},
|
||||
},
|
||||
{}, {}, {}
|
||||
};
|
||||
|
||||
auto result = expand_repeats(grammar);
|
||||
|
||||
AssertThat(result.variables, Equals(vector<Variable>({
|
||||
Variable("rule0", VariableTypeNamed, i_sym(2)),
|
||||
Variable("rule1", VariableTypeNamed, i_sym(3)),
|
||||
Variable("rule0_repeat1", VariableTypeAuxiliary, choice({
|
||||
AssertThat(result.variables, Equals(vector<Variable>{
|
||||
Variable{"rule0", VariableTypeNamed, i_sym(2)},
|
||||
Variable{"rule1", VariableTypeNamed, i_sym(3)},
|
||||
Variable{"rule0_repeat1", VariableTypeAuxiliary, choice({
|
||||
seq({ i_sym(2), i_token(10) }),
|
||||
i_token(10),
|
||||
})),
|
||||
Variable("rule1_repeat1", VariableTypeAuxiliary, choice({
|
||||
})},
|
||||
Variable{"rule1_repeat1", VariableTypeAuxiliary, choice({
|
||||
seq({ i_sym(3), i_token(11) }),
|
||||
i_token(11),
|
||||
})),
|
||||
})));
|
||||
})},
|
||||
}));
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -15,9 +15,9 @@ describe("expand_tokens", []() {
|
|||
|
||||
describe("string rules", [&]() {
|
||||
it("replaces strings with sequences of character sets", [&]() {
|
||||
LexicalGrammar grammar {
|
||||
LexicalGrammar grammar{
|
||||
{
|
||||
LexicalVariable {
|
||||
LexicalVariable{
|
||||
"rule_A",
|
||||
VariableTypeNamed,
|
||||
seq({
|
||||
|
|
@ -34,8 +34,8 @@ describe("expand_tokens", []() {
|
|||
auto result = expand_tokens(grammar);
|
||||
|
||||
AssertThat(result.second, Equals(CompileError::none()));
|
||||
AssertThat(result.first.variables, Equals(vector<LexicalVariable> {
|
||||
LexicalVariable {
|
||||
AssertThat(result.first.variables, Equals(vector<LexicalVariable>{
|
||||
LexicalVariable{
|
||||
"rule_A",
|
||||
VariableTypeNamed,
|
||||
seq({
|
||||
|
|
@ -53,9 +53,9 @@ describe("expand_tokens", []() {
|
|||
});
|
||||
|
||||
it("handles strings containing non-ASCII UTF8 characters", [&]() {
|
||||
LexicalGrammar grammar {
|
||||
LexicalGrammar grammar{
|
||||
{
|
||||
LexicalVariable {
|
||||
LexicalVariable{
|
||||
"rule_A",
|
||||
VariableTypeNamed,
|
||||
str("\u03B1 \u03B2"),
|
||||
|
|
@ -67,8 +67,8 @@ describe("expand_tokens", []() {
|
|||
|
||||
auto result = expand_tokens(grammar);
|
||||
|
||||
AssertThat(result.first.variables, Equals(vector<LexicalVariable> {
|
||||
LexicalVariable {
|
||||
AssertThat(result.first.variables, Equals(vector<LexicalVariable>{
|
||||
LexicalVariable{
|
||||
"rule_A",
|
||||
VariableTypeNamed,
|
||||
metadata(seq({
|
||||
|
|
@ -84,9 +84,9 @@ describe("expand_tokens", []() {
|
|||
|
||||
describe("regexp rules", [&]() {
|
||||
it("replaces regexps with the equivalent rule tree", [&]() {
|
||||
LexicalGrammar grammar {
|
||||
LexicalGrammar grammar{
|
||||
{
|
||||
LexicalVariable {
|
||||
LexicalVariable{
|
||||
"rule_A",
|
||||
VariableTypeNamed,
|
||||
seq({
|
||||
|
|
@ -103,8 +103,8 @@ describe("expand_tokens", []() {
|
|||
auto result = expand_tokens(grammar);
|
||||
|
||||
AssertThat(result.second, Equals(CompileError::none()));
|
||||
AssertThat(result.first.variables, Equals(vector<LexicalVariable> {
|
||||
LexicalVariable {
|
||||
AssertThat(result.first.variables, Equals(vector<LexicalVariable>{
|
||||
LexicalVariable{
|
||||
"rule_A",
|
||||
VariableTypeNamed,
|
||||
seq({
|
||||
|
|
@ -118,9 +118,9 @@ describe("expand_tokens", []() {
|
|||
});
|
||||
|
||||
it("handles regexps containing non-ASCII UTF8 characters", [&]() {
|
||||
LexicalGrammar grammar {
|
||||
LexicalGrammar grammar{
|
||||
{
|
||||
LexicalVariable {
|
||||
LexicalVariable{
|
||||
"rule_A",
|
||||
VariableTypeNamed,
|
||||
pattern("[^\u03B1-\u03B4]*"),
|
||||
|
|
@ -132,8 +132,8 @@ describe("expand_tokens", []() {
|
|||
|
||||
auto result = expand_tokens(grammar);
|
||||
|
||||
AssertThat(result.first.variables, Equals(vector<LexicalVariable> {
|
||||
LexicalVariable {
|
||||
AssertThat(result.first.variables, Equals(vector<LexicalVariable>{
|
||||
LexicalVariable{
|
||||
"rule_A",
|
||||
VariableTypeNamed,
|
||||
repeat(character({ 945, 946, 947, 948 }, false)),
|
||||
|
|
@ -143,9 +143,9 @@ describe("expand_tokens", []() {
|
|||
});
|
||||
|
||||
it("returns an error when the grammar contains an invalid regex", [&]() {
|
||||
LexicalGrammar grammar {
|
||||
LexicalGrammar grammar{
|
||||
{
|
||||
LexicalVariable {
|
||||
LexicalVariable{
|
||||
"rule_A",
|
||||
VariableTypeNamed,
|
||||
seq({
|
||||
|
|
|
|||
|
|
@ -16,9 +16,9 @@ using prepare_grammar::InitialSyntaxGrammar;
|
|||
|
||||
describe("extract_tokens", []() {
|
||||
it("moves strings, patterns, and sub-rules marked as tokens into the lexical grammar", [&]() {
|
||||
auto result = extract_tokens(InternedGrammar {
|
||||
auto result = extract_tokens(InternedGrammar{
|
||||
{
|
||||
Variable("rule_A", VariableTypeNamed, repeat1(seq({
|
||||
Variable{"rule_A", VariableTypeNamed, repeat1(seq({
|
||||
str("ab"),
|
||||
pattern("cd*"),
|
||||
choice({
|
||||
|
|
@ -26,10 +26,10 @@ describe("extract_tokens", []() {
|
|||
i_sym(2),
|
||||
token(repeat1(choice({ str("ef"), str("gh") }))),
|
||||
}),
|
||||
}))),
|
||||
Variable("rule_B", VariableTypeNamed, pattern("ij+")),
|
||||
Variable("rule_C", VariableTypeNamed, choice({ str("kl"), blank() })),
|
||||
Variable("rule_D", VariableTypeNamed, repeat1(i_sym(3)))
|
||||
}))},
|
||||
Variable{"rule_B", VariableTypeNamed, pattern("ij+")},
|
||||
Variable{"rule_C", VariableTypeNamed, choice({ str("kl"), blank() })},
|
||||
Variable{"rule_D", VariableTypeNamed, repeat1(i_sym(3))},
|
||||
},
|
||||
{},
|
||||
{},
|
||||
|
|
@ -42,8 +42,8 @@ describe("extract_tokens", []() {
|
|||
|
||||
AssertThat(error, Equals(CompileError::none()));
|
||||
|
||||
AssertThat(syntax_grammar.variables, Equals(vector<Variable>({
|
||||
Variable("rule_A", VariableTypeNamed, repeat1(seq({
|
||||
AssertThat(syntax_grammar.variables, Equals(vector<Variable>{
|
||||
Variable{"rule_A", VariableTypeNamed, repeat1(seq({
|
||||
|
||||
// This string is now the first token in the lexical grammar.
|
||||
i_token(0),
|
||||
|
|
@ -63,41 +63,41 @@ describe("extract_tokens", []() {
|
|||
// This token rule is now the third rule in the lexical grammar.
|
||||
i_token(2),
|
||||
}),
|
||||
}))),
|
||||
}))},
|
||||
|
||||
Variable("rule_C", VariableTypeNamed, choice({ i_token(4), blank() })),
|
||||
Variable("rule_D", VariableTypeNamed, repeat1(i_sym(2))),
|
||||
})));
|
||||
Variable{"rule_C", VariableTypeNamed, choice({ i_token(4), blank() })},
|
||||
Variable{"rule_D", VariableTypeNamed, repeat1(i_sym(2))},
|
||||
}));
|
||||
|
||||
AssertThat(lexical_grammar.variables, Equals(vector<LexicalVariable>({
|
||||
// Strings become anonymous rules.
|
||||
LexicalVariable("ab", VariableTypeAnonymous, str("ab"), true),
|
||||
LexicalVariable{"ab", VariableTypeAnonymous, str("ab"), true},
|
||||
|
||||
// Patterns become hidden rules.
|
||||
LexicalVariable("/cd*/", VariableTypeAuxiliary, pattern("cd*"), false),
|
||||
LexicalVariable{"/cd*/", VariableTypeAuxiliary, pattern("cd*"), false},
|
||||
|
||||
// Rules marked as tokens become hidden rules.
|
||||
LexicalVariable("/(ef|gh)*/", VariableTypeAuxiliary, repeat1(choice({
|
||||
LexicalVariable{"/(ef|gh)*/", VariableTypeAuxiliary, repeat1(choice({
|
||||
str("ef"),
|
||||
str("gh")
|
||||
})), false),
|
||||
})), false},
|
||||
|
||||
// This named rule was moved wholesale to the lexical grammar.
|
||||
LexicalVariable("rule_B", VariableTypeNamed, pattern("ij+"), false),
|
||||
LexicalVariable{"rule_B", VariableTypeNamed, pattern("ij+"), false},
|
||||
|
||||
// Strings become anonymous rules.
|
||||
LexicalVariable("kl", VariableTypeAnonymous, str("kl"), true),
|
||||
LexicalVariable{"kl", VariableTypeAnonymous, str("kl"), true},
|
||||
})));
|
||||
});
|
||||
|
||||
it("does not create duplicate tokens in the lexical grammar", [&]() {
|
||||
auto result = extract_tokens(InternedGrammar {
|
||||
auto result = extract_tokens(InternedGrammar{
|
||||
{
|
||||
Variable("rule_A", VariableTypeNamed, seq({
|
||||
Variable{"rule_A", VariableTypeNamed, seq({
|
||||
str("ab"),
|
||||
i_sym(0),
|
||||
str("ab"),
|
||||
})),
|
||||
})},
|
||||
},
|
||||
{},
|
||||
{},
|
||||
|
|
@ -118,18 +118,18 @@ describe("extract_tokens", []() {
|
|||
|
||||
it("does not move entire rules into the lexical grammar if their content is used elsewhere in the grammar", [&]() {
|
||||
auto result = extract_tokens(InternedGrammar{{
|
||||
Variable("rule_A", VariableTypeNamed, seq({ i_sym(1), str("ab") })),
|
||||
Variable("rule_B", VariableTypeNamed, str("cd")),
|
||||
Variable("rule_C", VariableTypeNamed, seq({ str("ef"), str("cd") })),
|
||||
Variable{"rule_A", VariableTypeNamed, seq({ i_sym(1), str("ab") })},
|
||||
Variable{"rule_B", VariableTypeNamed, str("cd")},
|
||||
Variable{"rule_C", VariableTypeNamed, seq({ str("ef"), str("cd") })},
|
||||
}, {}, {}, {}});
|
||||
|
||||
InitialSyntaxGrammar &syntax_grammar = get<0>(result);
|
||||
LexicalGrammar &lexical_grammar = get<1>(result);
|
||||
|
||||
AssertThat(syntax_grammar.variables, Equals(vector<Variable>({
|
||||
Variable("rule_A", VariableTypeNamed, seq({ i_sym(1), i_token(0) })),
|
||||
Variable("rule_B", VariableTypeNamed, i_token(1)),
|
||||
Variable("rule_C", VariableTypeNamed, seq({ i_token(2), i_token(1) })),
|
||||
Variable{"rule_A", VariableTypeNamed, seq({ i_sym(1), i_token(0) })},
|
||||
Variable{"rule_B", VariableTypeNamed, i_token(1)},
|
||||
Variable{"rule_C", VariableTypeNamed, seq({ i_token(2), i_token(1) })},
|
||||
})));
|
||||
|
||||
AssertThat(lexical_grammar.variables, Equals(vector<LexicalVariable> {
|
||||
|
|
@ -142,9 +142,9 @@ describe("extract_tokens", []() {
|
|||
it("renumbers the grammar's expected conflict symbols based on any moved rules", [&]() {
|
||||
auto result = extract_tokens(InternedGrammar{
|
||||
{
|
||||
Variable("rule_A", VariableTypeNamed, str("ok")),
|
||||
Variable("rule_B", VariableTypeNamed, repeat(i_sym(0))),
|
||||
Variable("rule_C", VariableTypeNamed, repeat(seq({ i_sym(0), i_sym(0) }))),
|
||||
Variable{"rule_A", VariableTypeNamed, str("ok")},
|
||||
Variable{"rule_B", VariableTypeNamed, repeat(i_sym(0))},
|
||||
Variable{"rule_C", VariableTypeNamed, repeat(seq({ i_sym(0), i_sym(0) }))},
|
||||
},
|
||||
{
|
||||
str(" ")
|
||||
|
|
@ -165,12 +165,17 @@ describe("extract_tokens", []() {
|
|||
|
||||
describe("handling extra tokens", [&]() {
|
||||
it("adds inline extra tokens to the lexical grammar's separators", [&]() {
|
||||
auto result = extract_tokens(InternedGrammar{{
|
||||
Variable("rule_A", VariableTypeNamed, str("x")),
|
||||
}, {
|
||||
str("y"),
|
||||
pattern("\\s+"),
|
||||
}, {}, {}});
|
||||
auto result = extract_tokens(InternedGrammar{
|
||||
{
|
||||
Variable{"rule_A", VariableTypeNamed, str("x")},
|
||||
},
|
||||
{
|
||||
str("y"),
|
||||
pattern("\\s+"),
|
||||
},
|
||||
{},
|
||||
{}
|
||||
});
|
||||
|
||||
AssertThat(get<2>(result), Equals(CompileError::none()));
|
||||
|
||||
|
|
@ -182,12 +187,17 @@ describe("extract_tokens", []() {
|
|||
});
|
||||
|
||||
it("handles inline extra tokens that match tokens in the grammar", [&]() {
|
||||
auto result = extract_tokens(InternedGrammar{{
|
||||
Variable("rule_A", VariableTypeNamed, str("x")),
|
||||
Variable("rule_B", VariableTypeNamed, str("y")),
|
||||
}, {
|
||||
str("y"),
|
||||
}, {}, {}});
|
||||
auto result = extract_tokens(InternedGrammar{
|
||||
{
|
||||
Variable{"rule_A", VariableTypeNamed, str("x")},
|
||||
Variable{"rule_B", VariableTypeNamed, str("y")},
|
||||
},
|
||||
{
|
||||
str("y"),
|
||||
},
|
||||
{},
|
||||
{}
|
||||
});
|
||||
|
||||
AssertThat(get<2>(result), Equals(CompileError::none()));
|
||||
AssertThat(get<1>(result).separators.size(), Equals<size_t>(0));
|
||||
|
|
@ -195,13 +205,18 @@ describe("extract_tokens", []() {
|
|||
});
|
||||
|
||||
it("updates extra symbols according to the new symbol numbers", [&]() {
|
||||
auto result = extract_tokens(InternedGrammar{{
|
||||
Variable("rule_A", VariableTypeNamed, seq({ str("w"), str("x"), i_sym(1) })),
|
||||
Variable("rule_B", VariableTypeNamed, str("y")),
|
||||
Variable("rule_C", VariableTypeNamed, str("z")),
|
||||
}, {
|
||||
i_sym(2),
|
||||
}, {}, {}});
|
||||
auto result = extract_tokens(InternedGrammar{
|
||||
{
|
||||
Variable{"rule_A", VariableTypeNamed, seq({ str("w"), str("x"), i_sym(1) })},
|
||||
Variable{"rule_B", VariableTypeNamed, str("y")},
|
||||
Variable{"rule_C", VariableTypeNamed, str("z")},
|
||||
},
|
||||
{
|
||||
i_sym(2),
|
||||
},
|
||||
{},
|
||||
{}
|
||||
});
|
||||
|
||||
AssertThat(get<2>(result), Equals(CompileError::none()));
|
||||
|
||||
|
|
@ -214,8 +229,8 @@ describe("extract_tokens", []() {
|
|||
|
||||
it("returns an error if any extra tokens are non-token symbols", [&]() {
|
||||
auto result = extract_tokens(InternedGrammar{{
|
||||
Variable("rule_A", VariableTypeNamed, seq({ str("x"), i_sym(1) })),
|
||||
Variable("rule_B", VariableTypeNamed, seq({ str("y"), str("z") })),
|
||||
Variable{"rule_A", VariableTypeNamed, seq({ str("x"), i_sym(1) })},
|
||||
Variable{"rule_B", VariableTypeNamed, seq({ str("y"), str("z") })},
|
||||
}, { i_sym(1) }, {}, {}});
|
||||
|
||||
AssertThat(get<2>(result), !Equals(CompileError::none()));
|
||||
|
|
@ -226,8 +241,8 @@ describe("extract_tokens", []() {
|
|||
|
||||
it("returns an error if any extra tokens are non-token rules", [&]() {
|
||||
auto result = extract_tokens(InternedGrammar{{
|
||||
Variable("rule_A", VariableTypeNamed, str("x")),
|
||||
Variable("rule_B", VariableTypeNamed, str("y")),
|
||||
Variable{"rule_A", VariableTypeNamed, str("x")},
|
||||
Variable{"rule_B", VariableTypeNamed, str("y")},
|
||||
}, { choice({ i_sym(1), blank() }) }, {}, {}});
|
||||
|
||||
AssertThat(get<2>(result), !Equals(CompileError::none()));
|
||||
|
|
@ -241,8 +256,8 @@ describe("extract_tokens", []() {
|
|||
it("returns an error if an external token has the same name as a non-terminal rule", [&]() {
|
||||
auto result = extract_tokens(InternedGrammar{
|
||||
{
|
||||
Variable("rule_A", VariableTypeNamed, seq({ str("x"), i_sym(1) })),
|
||||
Variable("rule_B", VariableTypeNamed, seq({ str("y"), str("z") })),
|
||||
Variable{"rule_A", VariableTypeNamed, seq({ str("x"), i_sym(1) })},
|
||||
Variable{"rule_B", VariableTypeNamed, seq({ str("y"), str("z") })},
|
||||
},
|
||||
{},
|
||||
{},
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ using prepare_grammar::flatten_rule;
|
|||
|
||||
describe("flatten_grammar", []() {
|
||||
it("associates each symbol with the precedence and associativity binding it to its successor", [&]() {
|
||||
SyntaxVariable result = flatten_rule(Variable(
|
||||
SyntaxVariable result = flatten_rule(Variable{
|
||||
"test",
|
||||
VariableTypeNamed,
|
||||
seq({
|
||||
|
|
@ -30,7 +30,7 @@ describe("flatten_grammar", []() {
|
|||
})),
|
||||
i_sym(7),
|
||||
})
|
||||
));
|
||||
});
|
||||
|
||||
AssertThat(result.name, Equals("test"));
|
||||
AssertThat(result.type, Equals(VariableTypeNamed));
|
||||
|
|
@ -54,14 +54,14 @@ describe("flatten_grammar", []() {
|
|||
});
|
||||
|
||||
it("uses the last assigned precedence", [&]() {
|
||||
SyntaxVariable result = flatten_rule(Variable(
|
||||
SyntaxVariable result = flatten_rule(Variable{
|
||||
"test1",
|
||||
VariableTypeNamed,
|
||||
prec_left(101, seq({
|
||||
i_sym(1),
|
||||
i_sym(2),
|
||||
}))
|
||||
));
|
||||
});
|
||||
|
||||
AssertThat(result.productions, Equals(vector<Production>({
|
||||
Production({
|
||||
|
|
@ -70,13 +70,13 @@ describe("flatten_grammar", []() {
|
|||
})
|
||||
})))
|
||||
|
||||
result = flatten_rule(Variable(
|
||||
result = flatten_rule(Variable{
|
||||
"test2",
|
||||
VariableTypeNamed,
|
||||
prec_left(101, seq({
|
||||
i_sym(1),
|
||||
}))
|
||||
));
|
||||
});
|
||||
|
||||
AssertThat(result.productions, Equals(vector<Production>({
|
||||
Production({
|
||||
|
|
|
|||
|
|
@ -15,27 +15,32 @@ using prepare_grammar::intern_symbols;
|
|||
|
||||
describe("intern_symbols", []() {
|
||||
it("replaces named symbols with numerically-indexed symbols", [&]() {
|
||||
Grammar grammar{{
|
||||
{ "x", choice({ sym("y"), sym("_z") }) },
|
||||
{ "y", sym("_z") },
|
||||
{ "_z", str("stuff") }
|
||||
}, {}, {}, {}};
|
||||
Grammar grammar{
|
||||
{
|
||||
{"x", choice({ sym("y"), sym("_z") })},
|
||||
{"y", sym("_z")},
|
||||
{"_z", str("stuff")}
|
||||
}, {}, {}, {}
|
||||
};
|
||||
|
||||
auto result = intern_symbols(grammar);
|
||||
|
||||
AssertThat(result.second, Equals(CompileError::none()));
|
||||
AssertThat(result.first.variables, Equals(vector<Variable>({
|
||||
Variable("x", VariableTypeNamed, choice({ i_sym(1), i_sym(2) })),
|
||||
Variable("y", VariableTypeNamed, i_sym(2)),
|
||||
Variable("_z", VariableTypeHidden, str("stuff")),
|
||||
})));
|
||||
AssertThat(result.first.variables, Equals(vector<Variable>{
|
||||
Variable{"x", VariableTypeNamed, choice({ i_sym(1), i_sym(2) })},
|
||||
Variable{"y", VariableTypeNamed, i_sym(2)},
|
||||
Variable{"_z", VariableTypeHidden, str("stuff")},
|
||||
}));
|
||||
});
|
||||
|
||||
describe("when there are symbols that reference undefined rules", [&]() {
|
||||
it("returns an error", []() {
|
||||
Grammar grammar{{
|
||||
{ "x", sym("y") },
|
||||
}, {}, {}, {}};
|
||||
Grammar grammar{
|
||||
{
|
||||
{"x", sym("y")},
|
||||
},
|
||||
{}, {}, {}
|
||||
};
|
||||
|
||||
auto result = intern_symbols(grammar);
|
||||
|
||||
|
|
@ -44,13 +49,17 @@ describe("intern_symbols", []() {
|
|||
});
|
||||
|
||||
it("translates the grammar's optional 'extra_tokens' to numerical symbols", [&]() {
|
||||
Grammar grammar{{
|
||||
{ "x", choice({ sym("y"), sym("z") }) },
|
||||
{ "y", sym("z") },
|
||||
{ "z", str("stuff") }
|
||||
}, {
|
||||
sym("z")
|
||||
}, {}, {}};
|
||||
Grammar grammar{
|
||||
{
|
||||
{"x", choice({ sym("y"), sym("z") })},
|
||||
{"y", sym("z")},
|
||||
{"z", str("stuff")}
|
||||
},
|
||||
{
|
||||
sym("z")
|
||||
},
|
||||
{}, {}
|
||||
};
|
||||
|
||||
auto result = intern_symbols(grammar);
|
||||
|
||||
|
|
@ -60,29 +69,34 @@ describe("intern_symbols", []() {
|
|||
});
|
||||
|
||||
it("records any rule names that match external token names", [&]() {
|
||||
Grammar grammar{{
|
||||
{ "x", choice({ sym("y"), sym("z") }) },
|
||||
{ "y", sym("z") },
|
||||
{ "z", str("stuff") }
|
||||
}, {}, {}, {
|
||||
"w",
|
||||
"z"
|
||||
}};
|
||||
Grammar grammar{
|
||||
{
|
||||
{"x", choice({ sym("y"), sym("z") })},
|
||||
{"y", sym("z")},
|
||||
{"z", str("stuff")},
|
||||
},
|
||||
{},
|
||||
{},
|
||||
{
|
||||
"w",
|
||||
"z"
|
||||
}
|
||||
};
|
||||
|
||||
auto result = intern_symbols(grammar);
|
||||
|
||||
AssertThat(result.first.external_tokens, Equals(vector<ExternalToken>({
|
||||
{
|
||||
AssertThat(result.first.external_tokens, Equals(vector<ExternalToken>{
|
||||
ExternalToken{
|
||||
"w",
|
||||
VariableTypeNamed,
|
||||
rules::NONE()
|
||||
},
|
||||
{
|
||||
ExternalToken{
|
||||
"z",
|
||||
VariableTypeNamed,
|
||||
Symbol(2, Symbol::NonTerminal)
|
||||
}
|
||||
})))
|
||||
},
|
||||
}))
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue