traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Collapse rules that contain only a single token

Browse source
This commit is contained in:
Max Brunsfeld 2014-01-05 01:19:32 -08:00
parent 04d18b56ed
commit 0985fa3008
7 changed files with 70 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

19
spec/compiler/grammar/prepare_grammar_spec.cpp
View file

@ -34,6 +34,25 @@ describe("preparing a grammar", []() {
rules::character('b') }) }
})));
});
it("turns entire rules into tokens when they contain no symbols", [&]() {
auto result = prepare_grammar(Grammar({
{ "rule1", sym("rule2") },
{ "rule2", seq({
character('a'),
character('b') }) }
}));
AssertThat(result.first, Equals(Grammar({
{ "rule1", sym("rule2") }
})));
AssertThat(result.second, Equals(Grammar("", {
{ "rule2", seq({
character('a'),
character('b') }) }
})));
});
});
END_TEST

76
spec/fixtures/parsers/arithmetic.c vendored
View file

@ -4,15 +4,13 @@
typedef enum {
ts_symbol_expression,
ts_symbol_term,
ts_symbol_number,
ts_symbol_factor,
ts_symbol_variable,
ts_symbol_6,
ts_symbol_5,
ts_symbol_4,
ts_symbol_3,
ts_symbol_2,
ts_symbol_1,
ts_symbol_2,
ts_symbol_number,
ts_symbol_variable,
ts_symbol___END__
} ts_symbol;
@ -20,23 +18,23 @@ static void ts_lex(TSParser *parser) {
START_LEXER();
switch (LEX_STATE()) {
case 0:
if (isdigit(LOOKAHEAD_CHAR()))
ADVANCE(3);
if (LOOKAHEAD_CHAR() == '(')
ADVANCE(2);
if (isalnum(LOOKAHEAD_CHAR()))
ADVANCE(3);
if (isdigit(LOOKAHEAD_CHAR()))
ADVANCE(2);
if (LOOKAHEAD_CHAR() == '(')
ADVANCE(1);
LEX_ERROR();
case 1:
if (isalnum(LOOKAHEAD_CHAR()))
ADVANCE(1);
ACCEPT_TOKEN(ts_symbol_1);
case 2:
ACCEPT_TOKEN(ts_symbol_2);
case 3:
if (isdigit(LOOKAHEAD_CHAR()))
ADVANCE(2);
ACCEPT_TOKEN(ts_symbol_number);
case 3:
if (isalnum(LOOKAHEAD_CHAR()))
ADVANCE(3);
ACCEPT_TOKEN(ts_symbol_4);
ACCEPT_TOKEN(ts_symbol_variable);
case 4:
LEX_ERROR();
case 5:
@ -44,19 +42,19 @@ static void ts_lex(TSParser *parser) {
ADVANCE(6);
LEX_ERROR();
case 6:
ACCEPT_TOKEN(ts_symbol_6);
ACCEPT_TOKEN(ts_symbol_4);
case 7:
if (LOOKAHEAD_CHAR() == '*')
ADVANCE(8);
LEX_ERROR();
case 8:
ACCEPT_TOKEN(ts_symbol_5);
ACCEPT_TOKEN(ts_symbol_3);
case 9:
if (LOOKAHEAD_CHAR() == ')')
ADVANCE(10);
LEX_ERROR();
case 10:
ACCEPT_TOKEN(ts_symbol_3);
ACCEPT_TOKEN(ts_symbol_2);
default:
LEX_ERROR();
}
@ -69,8 +67,6 @@ TSTree ts_parse_arithmetic(const char *input) {
case 0:
SET_LEX_STATE(0);
switch (LOOKAHEAD_SYM()) {
case ts_symbol_4:
SHIFT(13);
case ts_symbol_variable:
SHIFT(8);
case ts_symbol_factor:
@ -78,8 +74,6 @@ TSTree ts_parse_arithmetic(const char *input) {
case ts_symbol_number:
SHIFT(8);
case ts_symbol_1:
SHIFT(12);
case ts_symbol_2:
SHIFT(9);
case ts_symbol_term:
SHIFT(2);
@ -99,7 +93,7 @@ TSTree ts_parse_arithmetic(const char *input) {
case 2:
SET_LEX_STATE(5);
switch (LOOKAHEAD_SYM()) {
case ts_symbol_6:
case ts_symbol_4:
SHIFT(3);
default:
REDUCE(ts_symbol_expression, 1);
@ -107,20 +101,16 @@ TSTree ts_parse_arithmetic(const char *input) {
case 3:
SET_LEX_STATE(0);
switch (LOOKAHEAD_SYM()) {
case ts_symbol_4:
SHIFT(13);
case ts_symbol_1:
SHIFT(12);
case ts_symbol_term:
SHIFT(4);
case ts_symbol_2:
SHIFT(9);
case ts_symbol_variable:
SHIFT(8);
case ts_symbol_factor:
SHIFT(5);
case ts_symbol_1:
SHIFT(9);
case ts_symbol_number:
SHIFT(8);
case ts_symbol_term:
SHIFT(4);
default:
PARSE_ERROR();
}
@ -133,7 +123,7 @@ TSTree ts_parse_arithmetic(const char *input) {
case 5:
SET_LEX_STATE(7);
switch (LOOKAHEAD_SYM()) {
case ts_symbol_5:
case ts_symbol_3:
SHIFT(6);
default:
REDUCE(ts_symbol_term, 1);
@ -141,12 +131,8 @@ TSTree ts_parse_arithmetic(const char *input) {
case 6:
SET_LEX_STATE(0);
switch (LOOKAHEAD_SYM()) {
case ts_symbol_4:
SHIFT(13);
case ts_symbol_2:
SHIFT(9);
case ts_symbol_1:
SHIFT(12);
SHIFT(9);
case ts_symbol_number:
SHIFT(8);
case ts_symbol_variable:
@ -171,8 +157,6 @@ TSTree ts_parse_arithmetic(const char *input) {
case 9:
SET_LEX_STATE(0);
switch (LOOKAHEAD_SYM()) {
case ts_symbol_4:
SHIFT(13);
case ts_symbol_variable:
SHIFT(8);
case ts_symbol_factor:
@ -180,8 +164,6 @@ TSTree ts_parse_arithmetic(const char *input) {
case ts_symbol_number:
SHIFT(8);
case ts_symbol_1:
SHIFT(12);
case ts_symbol_2:
SHIFT(9);
case ts_symbol_term:
SHIFT(2);
@ -193,7 +175,7 @@ TSTree ts_parse_arithmetic(const char *input) {
case 10:
SET_LEX_STATE(9);
switch (LOOKAHEAD_SYM()) {
case ts_symbol_3:
case ts_symbol_2:
SHIFT(11);
default:
PARSE_ERROR();
@ -204,18 +186,6 @@ TSTree ts_parse_arithmetic(const char *input) {
default:
REDUCE(ts_symbol_factor, 3);
}
case 12:
SET_LEX_STATE(4);
switch (LOOKAHEAD_SYM()) {
default:
REDUCE(ts_symbol_variable, 1);
}
case 13:
SET_LEX_STATE(4);
switch (LOOKAHEAD_SYM()) {
default:
REDUCE(ts_symbol_number, 1);
}
default:
PARSE_ERROR();
}