traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

When removing duplicate lex states, update the error state too

Browse source 
Now, instead of being stored as a separate field on the parse table, the error
state is just the first state in the states vector.
This commit is contained in:
Max Brunsfeld 2015-12-29 20:58:50 -08:00
parent 36cdb3afcd
commit 939476c947
11 changed files with 11916 additions and 12342 deletions
Download patch file Download diff file Expand all files Collapse all files

52
spec/fixtures/parsers/anonymous_tokens.c vendored
View file

@ -35,19 +35,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
START_LEXER();
switch (lex_state) {
case 1:
START_TOKEN();
if ((lookahead == '\t') ||
(lookahead == ' '))
ADVANCE(1);
if (lookahead == '\n')
ADVANCE(2);
if (lookahead == '\r')
ADVANCE(3);
if (lookahead == '\"')
ADVANCE(4);
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(11);
LEX_ERROR();
ACCEPT_TOKEN(ts_builtin_sym_end);
case 2:
START_TOKEN();
ACCEPT_TOKEN(anon_sym_LF);
@ -84,23 +72,9 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
ACCEPT_TOKEN(aux_sym_SLASH_BSLASHd_SLASH);
case 12:
START_TOKEN();
if (lookahead == 0)
ADVANCE(13);
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(12);
LEX_ERROR();
case 13:
ACCEPT_TOKEN(ts_builtin_sym_end);
case 14:
START_TOKEN();
if (lookahead == 0)
ADVANCE(13);
if ((lookahead == '\t') ||
(lookahead == ' '))
ADVANCE(14);
if (lookahead == '\n')
ADVANCE(2);
if (lookahead == '\r')
@ -110,17 +84,27 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(11);
LEX_ERROR();
case 13:
START_TOKEN();
if (lookahead == 0)
ADVANCE(1);
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(13);
LEX_ERROR();
case ts_lex_state_error:
START_TOKEN();
if (lookahead == 0)
ADVANCE(13);
ADVANCE(1);
if ((lookahead == '\t') ||
(lookahead == ' '))
ADVANCE(14);
ADVANCE(0);
if (lookahead == '\n')
ADVANCE(15);
ADVANCE(2);
if (lookahead == '\r')
ADVANCE(16);
ADVANCE(3);
if (lookahead == '\"')
ADVANCE(4);
if ('0' <= lookahead && lookahead <= '9')
@ -132,9 +116,9 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
}
static TSStateId ts_lex_states[STATE_COUNT] = {
[0] = 1,
[1] = 12,
[2] = 12,
[0] = 12,
[1] = 13,
[2] = 13,
};
#pragma GCC diagnostic push

270
spec/fixtures/parsers/arithmetic.c vendored
View file

@ -74,23 +74,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
START_LEXER();
switch (lex_state) {
case 1:
START_TOKEN();
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(1);
if (lookahead == '#')
ADVANCE(2);
if (lookahead == '(')
ADVANCE(3);
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(4);
if (('A' <= lookahead && lookahead <= 'Z') ||
('a' <= lookahead && lookahead <= 'z') ||
(945 <= lookahead && lookahead <= 969))
ADVANCE(5);
LEX_ERROR();
ACCEPT_TOKEN(ts_builtin_sym_end);
case 2:
if (!((lookahead == 0) ||
(lookahead == '\n')))
@ -99,65 +83,63 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
case 3:
ACCEPT_TOKEN(anon_sym_LPAREN);
case 4:
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(4);
ACCEPT_TOKEN(sym_number);
ACCEPT_TOKEN(anon_sym_RPAREN);
case 5:
ACCEPT_TOKEN(anon_sym_STAR);
case 6:
ACCEPT_TOKEN(anon_sym_PLUS);
case 7:
ACCEPT_TOKEN(anon_sym_DASH);
case 8:
ACCEPT_TOKEN(anon_sym_SLASH);
case 9:
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(5);
ADVANCE(9);
ACCEPT_TOKEN(sym_number);
case 10:
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(10);
if (('A' <= lookahead && lookahead <= 'Z') ||
('a' <= lookahead && lookahead <= 'z') ||
(945 <= lookahead && lookahead <= 969))
ADVANCE(5);
ACCEPT_TOKEN(sym_variable);
case 6:
START_TOKEN();
if (lookahead == 0)
ADVANCE(7);
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(6);
if (lookahead == '#')
ADVANCE(2);
LEX_ERROR();
case 7:
ACCEPT_TOKEN(ts_builtin_sym_end);
case 8:
START_TOKEN();
if (lookahead == 0)
ADVANCE(7);
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(8);
if (lookahead == '#')
ADVANCE(2);
if (lookahead == '*')
ADVANCE(9);
if (lookahead == '+')
ADVANCE(10);
if (lookahead == '-')
ADVANCE(11);
if (lookahead == '/')
ADVANCE(12);
if (lookahead == '^')
ADVANCE(13);
LEX_ERROR();
case 9:
ACCEPT_TOKEN(anon_sym_STAR);
case 10:
ACCEPT_TOKEN(anon_sym_PLUS);
ACCEPT_TOKEN(sym_variable);
case 11:
ACCEPT_TOKEN(anon_sym_DASH);
case 12:
ACCEPT_TOKEN(anon_sym_SLASH);
case 13:
ACCEPT_TOKEN(anon_sym_CARET);
case 12:
START_TOKEN();
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(12);
if (lookahead == '#')
ADVANCE(2);
if (lookahead == '(')
ADVANCE(3);
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(9);
if (('A' <= lookahead && lookahead <= 'Z') ||
('a' <= lookahead && lookahead <= 'z') ||
(945 <= lookahead && lookahead <= 969))
ADVANCE(10);
LEX_ERROR();
case 13:
START_TOKEN();
if (lookahead == 0)
ADVANCE(1);
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(13);
if (lookahead == '#')
ADVANCE(2);
LEX_ERROR();
case 14:
START_TOKEN();
if (lookahead == 0)
ADVANCE(1);
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
@ -165,21 +147,39 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
ADVANCE(14);
if (lookahead == '#')
ADVANCE(2);
if (lookahead == ')')
ADVANCE(15);
if (lookahead == '*')
ADVANCE(9);
ADVANCE(5);
if (lookahead == '+')
ADVANCE(10);
ADVANCE(6);
if (lookahead == '-')
ADVANCE(11);
ADVANCE(7);
if (lookahead == '/')
ADVANCE(12);
ADVANCE(8);
if (lookahead == '^')
ADVANCE(13);
ADVANCE(11);
LEX_ERROR();
case 15:
ACCEPT_TOKEN(anon_sym_RPAREN);
START_TOKEN();
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(15);
if (lookahead == '#')
ADVANCE(2);
if (lookahead == ')')
ADVANCE(4);
if (lookahead == '*')
ADVANCE(5);
if (lookahead == '+')
ADVANCE(6);
if (lookahead == '-')
ADVANCE(7);
if (lookahead == '/')
ADVANCE(8);
if (lookahead == '^')
ADVANCE(11);
LEX_ERROR();
case 16:
START_TOKEN();
if ((lookahead == '\t') ||
@ -190,71 +190,39 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
if (lookahead == '#')
ADVANCE(2);
if (lookahead == ')')
ADVANCE(15);
LEX_ERROR();
case 17:
START_TOKEN();
if (lookahead == 0)
ADVANCE(7);
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(17);
if (lookahead == '#')
ADVANCE(2);
if (lookahead == '(')
ADVANCE(3);
if (lookahead == ')')
ADVANCE(15);
if (lookahead == '*')
ADVANCE(9);
if (lookahead == '+')
ADVANCE(10);
if (lookahead == '-')
ADVANCE(11);
if (lookahead == '/')
ADVANCE(12);
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(4);
if (('A' <= lookahead && lookahead <= 'Z') ||
('a' <= lookahead && lookahead <= 'z') ||
(945 <= lookahead && lookahead <= 969))
ADVANCE(5);
if (lookahead == '^')
ADVANCE(13);
LEX_ERROR();
case ts_lex_state_error:
START_TOKEN();
if (lookahead == 0)
ADVANCE(7);
ADVANCE(1);
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(17);
ADVANCE(0);
if (lookahead == '#')
ADVANCE(2);
if (lookahead == '(')
ADVANCE(3);
if (lookahead == ')')
ADVANCE(15);
if (lookahead == '*')
ADVANCE(9);
if (lookahead == '+')
ADVANCE(10);
if (lookahead == '-')
ADVANCE(11);
if (lookahead == '/')
ADVANCE(12);
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(4);
if (lookahead == '*')
ADVANCE(5);
if (lookahead == '+')
ADVANCE(6);
if (lookahead == '-')
ADVANCE(7);
if (lookahead == '/')
ADVANCE(8);
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(9);
if (('A' <= lookahead && lookahead <= 'Z') ||
('a' <= lookahead && lookahead <= 'z') ||
(945 <= lookahead && lookahead <= 969))
ADVANCE(5);
ADVANCE(10);
if (lookahead == '^')
ADVANCE(13);
ADVANCE(11);
LEX_ERROR();
default:
LEX_ERROR();
@ -262,39 +230,39 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
}
static TSStateId ts_lex_states[STATE_COUNT] = {
[0] = 1,
[1] = 6,
[2] = 8,
[3] = 8,
[4] = 1,
[5] = 14,
[6] = 14,
[0] = 12,
[1] = 13,
[2] = 14,
[3] = 14,
[4] = 12,
[5] = 15,
[6] = 15,
[7] = 16,
[8] = 1,
[9] = 14,
[8] = 12,
[9] = 15,
[10] = 16,
[11] = 14,
[12] = 1,
[13] = 1,
[14] = 1,
[15] = 1,
[16] = 1,
[17] = 14,
[18] = 14,
[19] = 14,
[20] = 14,
[21] = 14,
[22] = 8,
[23] = 1,
[24] = 1,
[25] = 1,
[26] = 1,
[27] = 1,
[28] = 8,
[29] = 8,
[30] = 8,
[31] = 8,
[32] = 8,
[11] = 15,
[12] = 12,
[13] = 12,
[14] = 12,
[15] = 12,
[16] = 12,
[17] = 15,
[18] = 15,
[19] = 15,
[20] = 15,
[21] = 15,
[22] = 14,
[23] = 12,
[24] = 12,
[25] = 12,
[26] = 12,
[27] = 12,
[28] = 14,
[29] = 14,
[30] = 14,
[31] = 14,
[32] = 14,
};
#pragma GCC diagnostic push

3526
spec/fixtures/parsers/c.c vendored
View file

File diff suppressed because it is too large Load diff

4968
spec/fixtures/parsers/cpp.c vendored
View file

File diff suppressed because it is too large Load diff

3858
spec/fixtures/parsers/golang.c vendored
View file

File diff suppressed because it is too large Load diff

11159
spec/fixtures/parsers/javascript.c vendored
View file

File diff suppressed because it is too large Load diff

374
spec/fixtures/parsers/json.c vendored
View file

@ -68,27 +68,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
START_LEXER();
switch (lex_state) {
case 1:
START_TOKEN();
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(1);
if (lookahead == '\"')
ADVANCE(2);
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(6);
if (lookahead == '[')
ADVANCE(9);
if (lookahead == 'f')
ADVANCE(10);
if (lookahead == 'n')
ADVANCE(15);
if (lookahead == 't')
ADVANCE(19);
if (lookahead == '{')
ADVANCE(23);
LEX_ERROR();
ACCEPT_TOKEN(ts_builtin_sym_end);
case 2:
if (lookahead == '\"')
ADVANCE(3);
@ -122,93 +102,75 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
ADVANCE(2);
ACCEPT_TOKEN(sym_string);
case 6:
if (lookahead == '.')
ADVANCE(7);
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(6);
ACCEPT_TOKEN(sym_number);
ACCEPT_TOKEN(anon_sym_COMMA);
case 7:
if ('0' <= lookahead && lookahead <= '9')
if (lookahead == '.')
ADVANCE(8);
LEX_ERROR();
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(7);
ACCEPT_TOKEN(sym_number);
case 8:
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(8);
ACCEPT_TOKEN(sym_number);
ADVANCE(9);
LEX_ERROR();
case 9:
ACCEPT_TOKEN(anon_sym_LBRACK);
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(9);
ACCEPT_TOKEN(sym_number);
case 10:
if (lookahead == 'a')
ADVANCE(11);
LEX_ERROR();
ACCEPT_TOKEN(anon_sym_COLON);
case 11:
if (lookahead == 'l')
ADVANCE(12);
LEX_ERROR();
ACCEPT_TOKEN(anon_sym_LBRACK);
case 12:
if (lookahead == 's')
ADVANCE(13);
LEX_ERROR();
ACCEPT_TOKEN(anon_sym_RBRACK);
case 13:
if (lookahead == 'e')
if (lookahead == 'a')
ADVANCE(14);
LEX_ERROR();
case 14:
ACCEPT_TOKEN(sym_false);
if (lookahead == 'l')
ADVANCE(15);
LEX_ERROR();
case 15:
if (lookahead == 'u')
if (lookahead == 's')
ADVANCE(16);
LEX_ERROR();
case 16:
if (lookahead == 'l')
if (lookahead == 'e')
ADVANCE(17);
LEX_ERROR();
case 17:
if (lookahead == 'l')
ADVANCE(18);
LEX_ERROR();
ACCEPT_TOKEN(sym_false);
case 18:
ACCEPT_TOKEN(sym_null);
if (lookahead == 'u')
ADVANCE(19);
LEX_ERROR();
case 19:
if (lookahead == 'r')
if (lookahead == 'l')
ADVANCE(20);
LEX_ERROR();
case 20:
if (lookahead == 'u')
if (lookahead == 'l')
ADVANCE(21);
LEX_ERROR();
case 21:
if (lookahead == 'e')
ADVANCE(22);
LEX_ERROR();
ACCEPT_TOKEN(sym_null);
case 22:
ACCEPT_TOKEN(sym_true);
if (lookahead == 'r')
ADVANCE(23);
LEX_ERROR();
case 23:
ACCEPT_TOKEN(anon_sym_LBRACE);
case 24:
START_TOKEN();
if (lookahead == 0)
ADVANCE(25);
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
if (lookahead == 'u')
ADVANCE(24);
LEX_ERROR();
case 25:
ACCEPT_TOKEN(ts_builtin_sym_end);
case 26:
START_TOKEN();
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(26);
if (lookahead == '\"')
ADVANCE(2);
if (lookahead == '}')
ADVANCE(27);
case 24:
if (lookahead == 'e')
ADVANCE(25);
LEX_ERROR();
case 25:
ACCEPT_TOKEN(sym_true);
case 26:
ACCEPT_TOKEN(anon_sym_LBRACE);
case 27:
ACCEPT_TOKEN(anon_sym_RBRACE);
case 28:
@ -221,22 +183,28 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
if (lookahead == '\"')
ADVANCE(2);
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(6);
ADVANCE(7);
if (lookahead == '[')
ADVANCE(9);
if (lookahead == ']')
ADVANCE(29);
ADVANCE(11);
if (lookahead == 'f')
ADVANCE(10);
ADVANCE(13);
if (lookahead == 'n')
ADVANCE(15);
ADVANCE(18);
if (lookahead == 't')
ADVANCE(19);
ADVANCE(22);
if (lookahead == '{')
ADVANCE(23);
ADVANCE(26);
LEX_ERROR();
case 29:
ACCEPT_TOKEN(anon_sym_RBRACK);
START_TOKEN();
if (lookahead == 0)
ADVANCE(1);
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(29);
LEX_ERROR();
case 30:
START_TOKEN();
if ((lookahead == '\t') ||
@ -244,13 +212,35 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(30);
if (lookahead == ',')
ADVANCE(31);
if (lookahead == ']')
ADVANCE(29);
if (lookahead == '\"')
ADVANCE(2);
if (lookahead == '}')
ADVANCE(27);
LEX_ERROR();
case 31:
ACCEPT_TOKEN(anon_sym_COMMA);
START_TOKEN();
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(31);
if (lookahead == '\"')
ADVANCE(2);
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(7);
if (lookahead == '[')
ADVANCE(11);
if (lookahead == ']')
ADVANCE(12);
if (lookahead == 'f')
ADVANCE(13);
if (lookahead == 'n')
ADVANCE(18);
if (lookahead == 't')
ADVANCE(22);
if (lookahead == '{')
ADVANCE(26);
LEX_ERROR();
case 32:
START_TOKEN();
if ((lookahead == '\t') ||
@ -258,8 +248,10 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(32);
if (lookahead == ',')
ADVANCE(6);
if (lookahead == ']')
ADVANCE(29);
ADVANCE(12);
LEX_ERROR();
case 33:
START_TOKEN();
@ -268,10 +260,8 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(33);
if (lookahead == ',')
ADVANCE(31);
if (lookahead == '}')
ADVANCE(27);
if (lookahead == ']')
ADVANCE(12);
LEX_ERROR();
case 34:
START_TOKEN();
@ -280,11 +270,21 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(34);
if (lookahead == ':')
ADVANCE(35);
if (lookahead == ',')
ADVANCE(6);
if (lookahead == '}')
ADVANCE(27);
LEX_ERROR();
case 35:
ACCEPT_TOKEN(anon_sym_COLON);
START_TOKEN();
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(35);
if (lookahead == ':')
ADVANCE(10);
LEX_ERROR();
case 36:
START_TOKEN();
if ((lookahead == '\t') ||
@ -305,67 +305,35 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
if (lookahead == '\"')
ADVANCE(2);
LEX_ERROR();
case 38:
START_TOKEN();
if (lookahead == 0)
ADVANCE(25);
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(38);
if (lookahead == '\"')
ADVANCE(2);
if (lookahead == ',')
ADVANCE(31);
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(6);
if (lookahead == ':')
ADVANCE(35);
if (lookahead == '[')
ADVANCE(9);
if (lookahead == ']')
ADVANCE(29);
if (lookahead == 'f')
ADVANCE(10);
if (lookahead == 'n')
ADVANCE(15);
if (lookahead == 't')
ADVANCE(19);
if (lookahead == '{')
ADVANCE(23);
if (lookahead == '}')
ADVANCE(27);
LEX_ERROR();
case ts_lex_state_error:
START_TOKEN();
if (lookahead == 0)
ADVANCE(25);
ADVANCE(1);
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(38);
ADVANCE(0);
if (lookahead == '\"')
ADVANCE(2);
if (lookahead == ',')
ADVANCE(31);
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(6);
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(7);
if (lookahead == ':')
ADVANCE(35);
if (lookahead == '[')
ADVANCE(9);
if (lookahead == ']')
ADVANCE(29);
if (lookahead == 'f')
ADVANCE(10);
if (lookahead == '[')
ADVANCE(11);
if (lookahead == ']')
ADVANCE(12);
if (lookahead == 'f')
ADVANCE(13);
if (lookahead == 'n')
ADVANCE(15);
ADVANCE(18);
if (lookahead == 't')
ADVANCE(19);
ADVANCE(22);
if (lookahead == '{')
ADVANCE(23);
ADVANCE(26);
if (lookahead == '}')
ADVANCE(27);
LEX_ERROR();
@ -375,75 +343,75 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
}
static TSStateId ts_lex_states[STATE_COUNT] = {
[0] = 1,
[1] = 24,
[2] = 24,
[3] = 26,
[4] = 28,
[5] = 30,
[6] = 30,
[7] = 26,
[8] = 28,
[9] = 24,
[10] = 30,
[11] = 30,
[12] = 32,
[13] = 1,
[14] = 30,
[15] = 30,
[16] = 32,
[17] = 30,
[18] = 33,
[19] = 30,
[20] = 34,
[21] = 1,
[22] = 33,
[23] = 33,
[24] = 26,
[25] = 28,
[26] = 30,
[27] = 33,
[28] = 32,
[29] = 33,
[30] = 33,
[31] = 33,
[32] = 33,
[33] = 34,
[34] = 1,
[35] = 33,
[0] = 28,
[1] = 29,
[2] = 29,
[3] = 30,
[4] = 31,
[5] = 32,
[6] = 32,
[7] = 30,
[8] = 31,
[9] = 29,
[10] = 32,
[11] = 32,
[12] = 33,
[13] = 28,
[14] = 32,
[15] = 32,
[16] = 33,
[17] = 32,
[18] = 34,
[19] = 32,
[20] = 35,
[21] = 28,
[22] = 34,
[23] = 34,
[24] = 30,
[25] = 31,
[26] = 32,
[27] = 34,
[28] = 33,
[29] = 34,
[30] = 34,
[31] = 34,
[32] = 34,
[33] = 35,
[34] = 28,
[35] = 34,
[36] = 36,
[37] = 37,
[38] = 33,
[39] = 33,
[40] = 34,
[41] = 1,
[42] = 33,
[38] = 34,
[39] = 34,
[40] = 35,
[41] = 28,
[42] = 34,
[43] = 36,
[44] = 36,
[45] = 33,
[45] = 34,
[46] = 36,
[47] = 33,
[48] = 33,
[47] = 34,
[48] = 34,
[49] = 36,
[50] = 30,
[51] = 30,
[50] = 32,
[51] = 32,
[52] = 36,
[53] = 30,
[54] = 30,
[55] = 32,
[56] = 24,
[57] = 24,
[58] = 33,
[59] = 24,
[60] = 34,
[61] = 1,
[62] = 33,
[53] = 32,
[54] = 32,
[55] = 33,
[56] = 29,
[57] = 29,
[58] = 34,
[59] = 29,
[60] = 35,
[61] = 28,
[62] = 34,
[63] = 36,
[64] = 24,
[65] = 24,
[64] = 29,
[65] = 29,
[66] = 36,
[67] = 24,
[68] = 24,
[67] = 29,
[68] = 29,
};
#pragma GCC diagnostic push

22
src/compiler/build_tables/build_lex_table.cc
View file

@ -48,15 +48,11 @@ class LexTableBuilder {
}
LexTable build() {
for (ParseState &parse_state : parse_table->states) {
LexItemSet item_set =
build_lex_item_set(parse_state.expected_inputs(), false);
parse_state.lex_state_id = add_lex_state(item_set);
}
add_lex_state(build_lex_item_set(parse_table->all_symbols(), true));
LexItemSet error_item_set =
build_lex_item_set(parse_table->all_symbols(), true);
populate_lex_state(error_item_set, LexTable::ERROR_STATE_ID);
for (ParseState &parse_state : parse_table->states)
parse_state.lex_state_id =
add_lex_state(build_lex_item_set(parse_state.expected_inputs(), false));
mark_fragile_tokens();
remove_duplicate_lex_states();
@ -109,19 +105,15 @@ class LexTableBuilder {
if (pair == lex_state_ids.end()) {
LexStateId state_id = lex_table.add_state();
lex_state_ids[item_set] = state_id;
populate_lex_state(item_set, state_id);
add_accept_token_actions(item_set, state_id);
add_advance_actions(item_set, state_id);
add_token_start(item_set, state_id);
return state_id;
} else {
return pair->second;
}
}
void populate_lex_state(const LexItemSet &item_set, LexStateId state_id) {
add_accept_token_actions(item_set, state_id);
add_advance_actions(item_set, state_id);
add_token_start(item_set, state_id);
}
void add_advance_actions(const LexItemSet &item_set, LexStateId state_id) {
for (const auto &transition : item_set.transitions()) {
const CharacterSet &rule = transition.first;

19
src/compiler/generate_code/c_code.cc
View file

@ -192,11 +192,11 @@ class CCodeGenerator {
indent([&]() {
line("START_LEXER();");
_switch("lex_state", [&]() {
for (size_t i = 0; i < lex_table.states.size(); i++)
_case(lex_state_index(i),
for (size_t i = 1; i < lex_table.states.size(); i++)
_case(to_string(i),
[&]() { add_lex_state(lex_table.states[i]); });
_case("ts_lex_state_error",
[&]() { add_lex_state(lex_table.error_state); });
[&]() { add_lex_state(lex_table.states[0]); });
_default([&]() { line("LEX_ERROR();"); });
});
});
@ -210,7 +210,7 @@ class CCodeGenerator {
size_t state_id = 0;
for (const auto &state : parse_table.states)
line("[" + to_string(state_id++) + "] = " +
lex_state_index(state.lex_state_id) + ",");
to_string(state.lex_state_id) + ",");
});
line("};");
line();
@ -308,7 +308,7 @@ class CCodeGenerator {
const set<rules::CharacterSet> &expected_inputs) {
switch (action.type) {
case LexActionTypeAdvance:
line("ADVANCE(" + lex_state_index(action.state_index) + ");");
line("ADVANCE(" + to_string(action.state_index) + ");");
break;
case LexActionTypeAccept:
line("ACCEPT_TOKEN(" + symbol_id(action.symbol) + ");");
@ -329,7 +329,8 @@ class CCodeGenerator {
indent([&]() {
for (const auto &pair : parse_actions) {
size_t index = pair.first;
line("[" + to_string(index) + "] = {.count = " + to_string(pair.second.size()) + "},");
line("[" + to_string(index) + "] = {.count = " +
to_string(pair.second.size()) + "},");
for (const ParseAction &action : pair.second) {
index++;
@ -367,7 +368,7 @@ class CCodeGenerator {
}
});
line ("};");
line("};");
}
size_t add_parse_actions(const vector<ParseAction> &actions) {
@ -396,10 +397,6 @@ class CCodeGenerator {
// Helper functions
string lex_state_index(size_t i) {
return to_string(i + 1);
}
string symbol_id(const rules::Symbol &symbol) {
if (symbol == rules::ERROR())
return "ts_builtin_sym_error";

7
src/compiler/lex_table.cc
View file

@ -70,12 +70,7 @@ LexStateId LexTable::add_state() {
}
LexState &LexTable::state(LexStateId id) {
if (id < 0)
return error_state;
else
return states[id];
return states[id];
}
const LexStateId LexTable::ERROR_STATE_ID = -1;
} // namespace tree_sitter

3
src/compiler/lex_table.h
View file

@ -69,12 +69,9 @@ typedef int64_t LexStateId;
class LexTable {
public:
static const LexStateId ERROR_STATE_ID;
LexStateId add_state();
LexState &state(LexStateId state_id);
std::vector<LexState> states;
LexState error_state;
};
} // namespace tree_sitter