traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove extra EOF actions in lexer

Browse source
This commit is contained in:
Max Brunsfeld 2014-04-04 08:44:35 -07:00
parent 32e94081fa
commit 129d2b9314
7 changed files with 1045 additions and 1153 deletions
Download patch file Download diff file Expand all files Collapse all files

31
examples/parsers/arithmetic.c
View file

@ -79,9 +79,8 @@ LEX_FN() {
ACCEPT_TOKEN(ts_builtin_sym_end);
case 2:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(2);
@ -92,8 +91,6 @@ LEX_FN() {
ACCEPT_TOKEN(ts_aux_sym_token2);
case 4:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -111,8 +108,6 @@ LEX_FN() {
ACCEPT_TOKEN(ts_aux_sym_token4);
case 7:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -134,8 +129,6 @@ LEX_FN() {
ACCEPT_TOKEN(ts_aux_sym_token6);
case 10:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -157,8 +150,6 @@ LEX_FN() {
ACCEPT_TOKEN(ts_aux_sym_token7);
case 12:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -186,8 +177,6 @@ LEX_FN() {
ACCEPT_TOKEN(ts_sym_variable);
case 16:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -203,8 +192,6 @@ LEX_FN() {
LEX_ERROR();
case 17:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -216,8 +203,6 @@ LEX_FN() {
LEX_ERROR();
case 18:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -231,8 +216,6 @@ LEX_FN() {
LEX_ERROR();
case 19:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -565,7 +548,7 @@ PARSE_TABLE = {
[1] = {
[ts_aux_sym_token3] = SHIFT(2),
[ts_aux_sym_token4] = SHIFT(100),
[ts_builtin_sym_end] = REDUCE(ts_sym_sum, 1),
[ts_builtin_sym_end] = REDUCE(ts_sym_difference, 1),
},
[2] = {
[ts_sym__operand1] = SHIFT(3),
@ -633,7 +616,7 @@ PARSE_TABLE = {
[ts_builtin_sym_error] = SHIFT(78),
},
[12] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_sum, 1),
[ts_aux_sym_token2] = REDUCE(ts_sym_difference, 1),
[ts_aux_sym_token3] = SHIFT(13),
[ts_aux_sym_token4] = SHIFT(76),
},
@ -653,7 +636,7 @@ PARSE_TABLE = {
[ts_aux_sym_token2] = REDUCE(ts_sym_sum, 3),
},
[15] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_product, 1),
[ts_aux_sym_token2] = REDUCE(ts_sym_quotient, 1),
[ts_aux_sym_token5] = SHIFT(16),
[ts_aux_sym_token6] = SHIFT(61),
},
@ -704,7 +687,7 @@ PARSE_TABLE = {
},
[23] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_quotient, 1),
[ts_aux_sym_token3] = REDUCE(ts_sym_product, 1),
[ts_aux_sym_token3] = REDUCE(ts_sym_quotient, 1),
[ts_aux_sym_token4] = REDUCE(ts_sym_quotient, 1),
[ts_aux_sym_token5] = SHIFT(24),
[ts_aux_sym_token6] = SHIFT(52),
@ -1224,7 +1207,7 @@ PARSE_TABLE = {
[ts_builtin_sym_end] = REDUCE(ts_sym_difference, 3),
},
[102] = {
[ts_aux_sym_token3] = REDUCE(ts_sym_quotient, 1),
[ts_aux_sym_token3] = REDUCE(ts_sym_product, 1),
[ts_aux_sym_token4] = REDUCE(ts_sym_product, 1),
[ts_aux_sym_token5] = SHIFT(103),
[ts_aux_sym_token6] = SHIFT(117),

70
examples/parsers/golang.c
View file

@ -227,9 +227,8 @@ LEX_FN() {
ACCEPT_TOKEN(ts_sym__var);
case 19:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(19);
@ -246,8 +245,6 @@ LEX_FN() {
ACCEPT_TOKEN(ts_sym__identifier);
case 21:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -297,9 +294,8 @@ LEX_FN() {
ACCEPT_TOKEN(ts_aux_sym_token2);
case 28:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(28);
@ -308,8 +304,6 @@ LEX_FN() {
LEX_ERROR();
case 29:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -338,9 +332,8 @@ LEX_FN() {
LEX_ERROR();
case 32:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(32);
@ -349,9 +342,8 @@ LEX_FN() {
LEX_ERROR();
case 33:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(33);
@ -362,9 +354,8 @@ LEX_FN() {
ACCEPT_TOKEN(ts_aux_sym_token5);
case 35:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(35);
@ -375,8 +366,6 @@ LEX_FN() {
ACCEPT_TOKEN(ts_aux_sym_token4);
case 37:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -577,8 +566,6 @@ LEX_FN() {
ACCEPT_TOKEN(ts_sym__struct);
case 59:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -590,8 +577,6 @@ LEX_FN() {
LEX_ERROR();
case 60:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -604,9 +589,8 @@ LEX_FN() {
LEX_ERROR();
case 61:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(61);
@ -617,8 +601,6 @@ LEX_FN() {
ACCEPT_TOKEN(ts_aux_sym_token8);
case 63:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -642,9 +624,8 @@ LEX_FN() {
LEX_ERROR();
case 64:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(64);
@ -653,8 +634,6 @@ LEX_FN() {
LEX_ERROR();
case 65:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -669,8 +648,6 @@ LEX_FN() {
LEX_ERROR();
case 66:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -685,8 +662,6 @@ LEX_FN() {
LEX_ERROR();
case 67:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -699,8 +674,6 @@ LEX_FN() {
LEX_ERROR();
case 68:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -715,9 +688,8 @@ LEX_FN() {
LEX_ERROR();
case 69:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(69);
@ -728,9 +700,8 @@ LEX_FN() {
ACCEPT_TOKEN(ts_aux_sym_token3);
case 71:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(71);
@ -763,9 +734,8 @@ LEX_FN() {
LEX_ERROR();
case 76:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(76);

2052
examples/parsers/javascript.c
View file

File diff suppressed because it is too large Load diff

30
examples/parsers/json.c
View file

@ -71,8 +71,6 @@ LEX_FN() {
ACCEPT_TOKEN(ts_builtin_sym_end);
case 2:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -88,9 +86,8 @@ LEX_FN() {
ACCEPT_TOKEN(ts_aux_sym_token4);
case 5:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(5);
@ -99,8 +96,6 @@ LEX_FN() {
LEX_ERROR();
case 6:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -114,9 +109,8 @@ LEX_FN() {
ACCEPT_TOKEN(ts_aux_sym_token6);
case 8:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(8);
@ -125,8 +119,6 @@ LEX_FN() {
LEX_ERROR();
case 9:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -248,9 +240,8 @@ LEX_FN() {
ACCEPT_TOKEN(ts_aux_sym_token1);
case 33:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(33);
@ -261,8 +252,6 @@ LEX_FN() {
ACCEPT_TOKEN(ts_aux_sym_token2);
case 35:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -274,8 +263,6 @@ LEX_FN() {
LEX_ERROR();
case 36:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
@ -299,9 +286,8 @@ LEX_FN() {
LEX_ERROR();
case 37:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(37);

4
spec/runtime/languages/javascript/main.txt
View file

@ -2,7 +2,7 @@
parses multiple statements
==========================================
var x = {}
firstFunction(x);
firstFunction(x)
secondFunction(x);
---
(program
@ -14,7 +14,7 @@ secondFunction(x);
parses if statements
==========================================
if (isReady()) {
console.log(theData);
console.log(theData)
}
---
(program

9
src/compiler/build_tables/build_tables.cc
View file

@ -88,11 +88,12 @@ namespace tree_sitter {
LexItemSet lex_item_set_for_parse_state(const ParseState &state) {
LexItemSet result;
for (auto &symbol : state.expected_inputs())
if (lex_grammar.has_definition(symbol)) {
for (auto &symbol : state.expected_inputs()) {
if (lex_grammar.has_definition(symbol))
result.insert(LexItem(symbol, after_separators(lex_grammar.rule(symbol))));
}
result.insert(LexItem(rules::END_OF_INPUT(), after_separators(CharacterSet({ 0 }).copy())));
if (symbol == rules::END_OF_INPUT())
result.insert(LexItem(symbol, after_separators(CharacterSet({ 0 }).copy())));
}
return result;
}

2
src/compiler/parse_table.cc
View file

@ -58,7 +58,7 @@ namespace tree_sitter {
set<Symbol> ParseState::expected_inputs() const {
set<Symbol> result;
for (auto pair : actions)
for (auto &pair : actions)
result.insert(pair.first);
return result;
}