traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactor bookkeeping of token starts in lexical rules

Browse source 
- Move lex items and parse items into their own files
This commit is contained in:
Max Brunsfeld 2014-04-17 13:20:43 -07:00
parent a12cd49585
commit 33d781f492
19 changed files with 280 additions and 232 deletions
Download patch file Download diff file Expand all files Collapse all files

28
spec/compiler/build_tables/conflict_manager_spec.cc
View file

@ -70,35 +70,35 @@ describe("resolving parse conflicts", []() {
describe("when the shift has higher precedence", [&]() {
ParseAction shift = ParseAction::Shift(2, { 3 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 1);
it("does not record a conflict", [&]() {
manager->resolve_parse_action(sym1, shift, reduce);
manager->resolve_parse_action(sym1, reduce, shift);
AssertThat(manager->conflicts(), IsEmpty());
});
it("favors the shift", [&]() {
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsTrue());
});
});
describe("when the reduce has higher precedence", [&]() {
ParseAction shift = ParseAction::Shift(2, { 1 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 3);
it("does not record a conflict", [&]() {
manager->resolve_parse_action(sym1, reduce, shift);
manager->resolve_parse_action(sym1, shift, reduce);
AssertThat(manager->conflicts(), IsEmpty());
});
it("favors the reduce", [&]() {
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsTrue());
});
});
describe("when the precedences are equal", [&]() {
ParseAction shift = ParseAction::Shift(2, { 0 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 0);
@ -110,17 +110,17 @@ describe("resolving parse conflicts", []() {
Conflict("rule1: shift (precedence 0) / reduce rule2 (precedence 0)")
})));
});
it("favors the shift", [&]() {
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsTrue());
});
});
describe("when the shift has conflicting precedences compared to the reduce", [&]() {
ParseAction shift = ParseAction::Shift(2, { 0, 1, 3 });
ParseAction reduce = ParseAction::Reduce(sym2, 1, 2);
it("records a conflict", [&]() {
manager->resolve_parse_action(sym1, reduce, shift);
manager->resolve_parse_action(sym1, shift, reduce);
@ -128,7 +128,7 @@ describe("resolving parse conflicts", []() {
Conflict("rule1: shift (precedence 0, 1, 3) / reduce rule2 (precedence 2)")
})));
});
it("favors the shift", [&]() {
AssertThat(manager->resolve_parse_action(sym1, shift, reduce), IsFalse());
AssertThat(manager->resolve_parse_action(sym1, reduce, shift), IsTrue());
@ -140,23 +140,23 @@ describe("resolving parse conflicts", []() {
describe("when one action has higher precedence", [&]() {
ParseAction left = ParseAction::Reduce(sym2, 1, 0);
ParseAction right = ParseAction::Reduce(sym2, 1, 3);
it("favors that action", [&]() {
AssertThat(manager->resolve_parse_action(sym1, left, right), IsTrue());
AssertThat(manager->resolve_parse_action(sym1, right, left), IsFalse());
});
it("does not record a conflict", [&]() {
manager->resolve_parse_action(sym1, left, right);
manager->resolve_parse_action(sym1, right, left);
AssertThat(manager->conflicts(), IsEmpty());
});
});
describe("when the actions have the same precedence", [&]() {
ParseAction left = ParseAction::Reduce(sym1, 1, 0);
ParseAction right = ParseAction::Reduce(sym2, 1, 0);
it("favors the symbol listed earlier in the grammar", [&]() {
AssertThat(manager->resolve_parse_action(sym1, right, left), IsTrue());
AssertThat(manager->resolve_parse_action(sym1, left, right), IsFalse());

43
spec/compiler/build_tables/get_metadata_spec.cc
View file

@ -33,49 +33,6 @@ describe("getting metadata for rules", []() {
AssertThat(get_metadata(rule, key1), Equals(0));
});
});
it("works for metadata rules preceded by other rules that can be blank", [&]() {
auto rule = seq({
repeat(sym("x")),
make_shared<Metadata>(sym("x"), map<MetadataKey, int>({
{ key1, 1 },
{ key2, 2 },
})),
});
AssertThat(get_metadata(rule, key2), Equals(2));
});
it("works for choices containing metadata rule", [&]() {
auto rule = choice({
sym("x"),
make_shared<Metadata>(sym("x"), map<MetadataKey, int>({
{ key1, 1 },
{ key2, 2 },
})),
});
AssertThat(get_metadata(rule, key2), Equals(1));
});
it("works for repetitions containing metadata rules", [&]() {
auto rule = repeat(make_shared<Metadata>(sym("x"), map<MetadataKey, int>({
{ key1, 1 },
{ key2, 2 },
})));
AssertThat(get_metadata(rule, key2), Equals(2));
});
it("returns 0 for metadata rules preceded by rules that can't be blank", [&]() {
auto rule = seq({
sym("x"),
make_shared<Metadata>(sym("y"), map<MetadataKey, int>({
{ key1, 1 },
{ key2, 2 },
})),
});
AssertThat(get_metadata(rule, key2), Equals(0));
});
});
END_TEST

43
spec/compiler/build_tables/lex_item_spec.cc Normal file
View file

@ -0,0 +1,43 @@
#include "compiler_spec_helper.h"
#include "compiler/build_tables/item_set_transitions.h"
#include "compiler/prepared_grammar.h"
using namespace rules;
using namespace build_tables;
START_TEST
describe("lex items", []() {
describe("determining if an item is the start of a token", [&]() {
Symbol sym("x");
rule_ptr token_start = make_shared<Metadata>(str("a"), map<MetadataKey, int>({
{ START_TOKEN, 1 }
}));
it("returns true for rules designated as token starts", [&]() {
LexItem item(sym, token_start);
AssertThat(item.is_token_start(), IsTrue());
});
it("returns false for rules not designated as token starts", [&]() {
AssertThat(LexItem(sym, make_shared<Metadata>(str("a"), map<MetadataKey, int>({
{ START_TOKEN, 0 }
}))).is_token_start(), IsFalse());
AssertThat(LexItem(sym, str("a")).is_token_start(), IsFalse());
});
describe("when given a sequence containing a token start", [&]() {
it("returns true when the rule before the token start may be blank", [&]() {
LexItem item(sym, seq({ repeat(str("a")), token_start }));
AssertThat(item.is_token_start(), IsTrue());
});
it("returns false when the rule before the token start cannot be blank", [&]() {
LexItem item(sym, seq({ str("a"), token_start }));
AssertThat(item.is_token_start(), IsFalse());
});
});
});
});
END_TEST

4
spec/compiler/build_tables/rule_transitions_spec.cc
View file

@ -183,12 +183,12 @@ describe("rule transitions", []() {
{ CharacterSet({ 'a' }), rule }
})));
});
it("preserves metadata", [&]() {
map<MetadataKey, int> metadata_value({
{ PRECEDENCE, 5 }
});
rule_ptr rule = make_shared<Metadata>(seq({ sym("x"), sym("y") }), metadata_value);
AssertThat(
sym_transitions(rule),