traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix symbol names for extracted tokens

Browse source
This commit is contained in:
Max Brunsfeld 2014-05-19 22:43:33 -07:00
parent 649f200831
commit c30055ba18
10 changed files with 17018 additions and 16970 deletions
Download patch file Download diff file Expand all files Collapse all files

274
examples/parsers/arithmetic.c
View file

@ -13,13 +13,13 @@ enum {
ts_sym_group,
ts_sym_number,
ts_sym_variable,
ts_aux_sym_token0,
ts_aux_sym_token1,
ts_aux_sym_token2,
ts_aux_sym_token3,
ts_aux_sym_token4,
ts_aux_sym_token5,
ts_aux_sym_token6,
ts_aux_sym_1,
ts_aux_sym_2,
ts_aux_sym_3,
ts_aux_sym_4,
ts_aux_sym_5,
ts_aux_sym_6,
ts_aux_sym_7,
};
SYMBOL_NAMES = {
@ -34,26 +34,26 @@ SYMBOL_NAMES = {
[ts_builtin_sym_end] = "end",
[ts_sym_number] = "number",
[ts_sym_variable] = "variable",
[ts_aux_sym_token0] = "",
[ts_aux_sym_token1] = "",
[ts_aux_sym_token2] = "",
[ts_aux_sym_token3] = "",
[ts_aux_sym_token4] = "",
[ts_aux_sym_token5] = "",
[ts_aux_sym_token6] = "",
[ts_aux_sym_1] = "'+'",
[ts_aux_sym_2] = "'-'",
[ts_aux_sym_3] = "'*'",
[ts_aux_sym_4] = "'/'",
[ts_aux_sym_5] = "'^'",
[ts_aux_sym_6] = "'('",
[ts_aux_sym_7] = "')'",
};
UBIQUITOUS_SYMBOLS = {
};
HIDDEN_SYMBOLS = {
[ts_aux_sym_token0] = 1,
[ts_aux_sym_token1] = 1,
[ts_aux_sym_token2] = 1,
[ts_aux_sym_token3] = 1,
[ts_aux_sym_token4] = 1,
[ts_aux_sym_token5] = 1,
[ts_aux_sym_token6] = 1,
[ts_aux_sym_1] = 1,
[ts_aux_sym_2] = 1,
[ts_aux_sym_3] = 1,
[ts_aux_sym_4] = 1,
[ts_aux_sym_5] = 1,
[ts_aux_sym_6] = 1,
[ts_aux_sym_7] = 1,
};
LEX_FN() {
@ -74,7 +74,7 @@ LEX_FN() {
ADVANCE(4);
LEX_ERROR();
case 2:
ACCEPT_TOKEN(ts_aux_sym_token5);
ACCEPT_TOKEN(ts_aux_sym_6);
case 3:
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(3);
@ -108,15 +108,15 @@ LEX_FN() {
case 6:
ACCEPT_TOKEN(ts_builtin_sym_end);
case 7:
ACCEPT_TOKEN(ts_aux_sym_token2);
ACCEPT_TOKEN(ts_aux_sym_3);
case 8:
ACCEPT_TOKEN(ts_aux_sym_token0);
ACCEPT_TOKEN(ts_aux_sym_1);
case 9:
ACCEPT_TOKEN(ts_aux_sym_token1);
ACCEPT_TOKEN(ts_aux_sym_2);
case 10:
ACCEPT_TOKEN(ts_aux_sym_token3);
ACCEPT_TOKEN(ts_aux_sym_4);
case 11:
ACCEPT_TOKEN(ts_aux_sym_token4);
ACCEPT_TOKEN(ts_aux_sym_5);
case 12:
START_TOKEN();
if (('\t' <= lookahead && lookahead <= '\n') ||
@ -137,7 +137,7 @@ LEX_FN() {
ADVANCE(11);
LEX_ERROR();
case 13:
ACCEPT_TOKEN(ts_aux_sym_token6);
ACCEPT_TOKEN(ts_aux_sym_7);
case 14:
START_TOKEN();
if ((lookahead == '\t') ||
@ -258,15 +258,15 @@ PARSE_TABLE = {
[ts_sym_group] = SHIFT(12),
[ts_sym_number] = SHIFT(12),
[ts_sym_variable] = SHIFT(12),
[ts_aux_sym_token5] = SHIFT(13),
[ts_aux_sym_6] = SHIFT(13),
},
[1] = {
[ts_builtin_sym_end] = ACCEPT_INPUT(),
[ts_aux_sym_token0] = SHIFT(2),
[ts_aux_sym_token1] = SHIFT(4),
[ts_aux_sym_token2] = SHIFT(6),
[ts_aux_sym_token3] = SHIFT(8),
[ts_aux_sym_token4] = SHIFT(10),
[ts_aux_sym_1] = SHIFT(2),
[ts_aux_sym_2] = SHIFT(4),
[ts_aux_sym_3] = SHIFT(6),
[ts_aux_sym_4] = SHIFT(8),
[ts_aux_sym_5] = SHIFT(10),
},
[2] = {
[ts_sym_expression] = SHIFT(3),
@ -278,15 +278,15 @@ PARSE_TABLE = {
[ts_sym_group] = SHIFT(12),
[ts_sym_number] = SHIFT(12),
[ts_sym_variable] = SHIFT(12),
[ts_aux_sym_token5] = SHIFT(13),
[ts_aux_sym_6] = SHIFT(13),
},
[3] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_sum, 3),
[ts_aux_sym_token0] = SHIFT(2),
[ts_aux_sym_token1] = SHIFT(4),
[ts_aux_sym_token2] = SHIFT(6),
[ts_aux_sym_token3] = SHIFT(8),
[ts_aux_sym_token4] = SHIFT(10),
[ts_aux_sym_1] = SHIFT(2),
[ts_aux_sym_2] = SHIFT(4),
[ts_aux_sym_3] = SHIFT(6),
[ts_aux_sym_4] = SHIFT(8),
[ts_aux_sym_5] = SHIFT(10),
},
[4] = {
[ts_sym_expression] = SHIFT(5),
@ -298,15 +298,15 @@ PARSE_TABLE = {
[ts_sym_group] = SHIFT(12),
[ts_sym_number] = SHIFT(12),
[ts_sym_variable] = SHIFT(12),
[ts_aux_sym_token5] = SHIFT(13),
[ts_aux_sym_6] = SHIFT(13),
},
[5] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_difference, 3),
[ts_aux_sym_token0] = SHIFT(2),
[ts_aux_sym_token1] = SHIFT(4),
[ts_aux_sym_token2] = SHIFT(6),
[ts_aux_sym_token3] = SHIFT(8),
[ts_aux_sym_token4] = SHIFT(10),
[ts_aux_sym_1] = SHIFT(2),
[ts_aux_sym_2] = SHIFT(4),
[ts_aux_sym_3] = SHIFT(6),
[ts_aux_sym_4] = SHIFT(8),
[ts_aux_sym_5] = SHIFT(10),
},
[6] = {
[ts_sym_expression] = SHIFT(7),
@ -318,15 +318,15 @@ PARSE_TABLE = {
[ts_sym_group] = SHIFT(12),
[ts_sym_number] = SHIFT(12),
[ts_sym_variable] = SHIFT(12),
[ts_aux_sym_token5] = SHIFT(13),
[ts_aux_sym_6] = SHIFT(13),
},
[7] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_product, 3),
[ts_aux_sym_token0] = REDUCE(ts_sym_product, 3),
[ts_aux_sym_token1] = REDUCE(ts_sym_product, 3),
[ts_aux_sym_token2] = SHIFT(6),
[ts_aux_sym_token3] = SHIFT(8),
[ts_aux_sym_token4] = SHIFT(10),
[ts_aux_sym_1] = REDUCE(ts_sym_product, 3),
[ts_aux_sym_2] = REDUCE(ts_sym_product, 3),
[ts_aux_sym_3] = SHIFT(6),
[ts_aux_sym_4] = SHIFT(8),
[ts_aux_sym_5] = SHIFT(10),
},
[8] = {
[ts_sym_expression] = SHIFT(9),
@ -338,15 +338,15 @@ PARSE_TABLE = {
[ts_sym_group] = SHIFT(12),
[ts_sym_number] = SHIFT(12),
[ts_sym_variable] = SHIFT(12),
[ts_aux_sym_token5] = SHIFT(13),
[ts_aux_sym_6] = SHIFT(13),
},
[9] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_quotient, 3),
[ts_aux_sym_token0] = REDUCE(ts_sym_quotient, 3),
[ts_aux_sym_token1] = REDUCE(ts_sym_quotient, 3),
[ts_aux_sym_token2] = SHIFT(6),
[ts_aux_sym_token3] = SHIFT(8),
[ts_aux_sym_token4] = SHIFT(10),
[ts_aux_sym_1] = REDUCE(ts_sym_quotient, 3),
[ts_aux_sym_2] = REDUCE(ts_sym_quotient, 3),
[ts_aux_sym_3] = SHIFT(6),
[ts_aux_sym_4] = SHIFT(8),
[ts_aux_sym_5] = SHIFT(10),
},
[10] = {
[ts_sym_expression] = SHIFT(11),
@ -358,23 +358,23 @@ PARSE_TABLE = {
[ts_sym_group] = SHIFT(12),
[ts_sym_number] = SHIFT(12),
[ts_sym_variable] = SHIFT(12),
[ts_aux_sym_token5] = SHIFT(13),
[ts_aux_sym_6] = SHIFT(13),
},
[11] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token0] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token1] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token2] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token3] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token4] = SHIFT(10),
[ts_aux_sym_1] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_2] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_3] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_4] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_5] = SHIFT(10),
},
[12] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token0] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token1] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token2] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token3] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token4] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_1] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_2] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_3] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_4] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_5] = REDUCE(ts_sym_expression, 1),
},
[13] = {
[ts_sym_expression] = SHIFT(14),
@ -387,15 +387,15 @@ PARSE_TABLE = {
[ts_builtin_sym_error] = SHIFT(31),
[ts_sym_number] = SHIFT(25),
[ts_sym_variable] = SHIFT(25),
[ts_aux_sym_token5] = SHIFT(26),
[ts_aux_sym_6] = SHIFT(26),
},
[14] = {
[ts_aux_sym_token0] = SHIFT(15),
[ts_aux_sym_token1] = SHIFT(17),
[ts_aux_sym_token2] = SHIFT(19),
[ts_aux_sym_token3] = SHIFT(21),
[ts_aux_sym_token4] = SHIFT(23),
[ts_aux_sym_token6] = SHIFT(30),
[ts_aux_sym_1] = SHIFT(15),
[ts_aux_sym_2] = SHIFT(17),
[ts_aux_sym_3] = SHIFT(19),
[ts_aux_sym_4] = SHIFT(21),
[ts_aux_sym_5] = SHIFT(23),
[ts_aux_sym_7] = SHIFT(30),
},
[15] = {
[ts_sym_expression] = SHIFT(16),
@ -407,15 +407,15 @@ PARSE_TABLE = {
[ts_sym_group] = SHIFT(25),
[ts_sym_number] = SHIFT(25),
[ts_sym_variable] = SHIFT(25),
[ts_aux_sym_token5] = SHIFT(26),
[ts_aux_sym_6] = SHIFT(26),
},
[16] = {
[ts_aux_sym_token0] = SHIFT(15),
[ts_aux_sym_token1] = SHIFT(17),
[ts_aux_sym_token2] = SHIFT(19),
[ts_aux_sym_token3] = SHIFT(21),
[ts_aux_sym_token4] = SHIFT(23),
[ts_aux_sym_token6] = REDUCE(ts_sym_sum, 3),
[ts_aux_sym_1] = SHIFT(15),
[ts_aux_sym_2] = SHIFT(17),
[ts_aux_sym_3] = SHIFT(19),
[ts_aux_sym_4] = SHIFT(21),
[ts_aux_sym_5] = SHIFT(23),
[ts_aux_sym_7] = REDUCE(ts_sym_sum, 3),
},
[17] = {
[ts_sym_expression] = SHIFT(18),
@ -427,15 +427,15 @@ PARSE_TABLE = {
[ts_sym_group] = SHIFT(25),
[ts_sym_number] = SHIFT(25),
[ts_sym_variable] = SHIFT(25),
[ts_aux_sym_token5] = SHIFT(26),
[ts_aux_sym_6] = SHIFT(26),
},
[18] = {
[ts_aux_sym_token0] = SHIFT(15),
[ts_aux_sym_token1] = SHIFT(17),
[ts_aux_sym_token2] = SHIFT(19),
[ts_aux_sym_token3] = SHIFT(21),
[ts_aux_sym_token4] = SHIFT(23),
[ts_aux_sym_token6] = REDUCE(ts_sym_difference, 3),
[ts_aux_sym_1] = SHIFT(15),
[ts_aux_sym_2] = SHIFT(17),
[ts_aux_sym_3] = SHIFT(19),
[ts_aux_sym_4] = SHIFT(21),
[ts_aux_sym_5] = SHIFT(23),
[ts_aux_sym_7] = REDUCE(ts_sym_difference, 3),
},
[19] = {
[ts_sym_expression] = SHIFT(20),
@ -447,15 +447,15 @@ PARSE_TABLE = {
[ts_sym_group] = SHIFT(25),
[ts_sym_number] = SHIFT(25),
[ts_sym_variable] = SHIFT(25),
[ts_aux_sym_token5] = SHIFT(26),
[ts_aux_sym_6] = SHIFT(26),
},
[20] = {
[ts_aux_sym_token0] = REDUCE(ts_sym_product, 3),
[ts_aux_sym_token1] = REDUCE(ts_sym_product, 3),
[ts_aux_sym_token2] = SHIFT(19),
[ts_aux_sym_token3] = SHIFT(21),
[ts_aux_sym_token4] = SHIFT(23),
[ts_aux_sym_token6] = REDUCE(ts_sym_product, 3),
[ts_aux_sym_1] = REDUCE(ts_sym_product, 3),
[ts_aux_sym_2] = REDUCE(ts_sym_product, 3),
[ts_aux_sym_3] = SHIFT(19),
[ts_aux_sym_4] = SHIFT(21),
[ts_aux_sym_5] = SHIFT(23),
[ts_aux_sym_7] = REDUCE(ts_sym_product, 3),
},
[21] = {
[ts_sym_expression] = SHIFT(22),
@ -467,15 +467,15 @@ PARSE_TABLE = {
[ts_sym_group] = SHIFT(25),
[ts_sym_number] = SHIFT(25),
[ts_sym_variable] = SHIFT(25),
[ts_aux_sym_token5] = SHIFT(26),
[ts_aux_sym_6] = SHIFT(26),
},
[22] = {
[ts_aux_sym_token0] = REDUCE(ts_sym_quotient, 3),
[ts_aux_sym_token1] = REDUCE(ts_sym_quotient, 3),
[ts_aux_sym_token2] = SHIFT(19),
[ts_aux_sym_token3] = SHIFT(21),
[ts_aux_sym_token4] = SHIFT(23),
[ts_aux_sym_token6] = REDUCE(ts_sym_quotient, 3),
[ts_aux_sym_1] = REDUCE(ts_sym_quotient, 3),
[ts_aux_sym_2] = REDUCE(ts_sym_quotient, 3),
[ts_aux_sym_3] = SHIFT(19),
[ts_aux_sym_4] = SHIFT(21),
[ts_aux_sym_5] = SHIFT(23),
[ts_aux_sym_7] = REDUCE(ts_sym_quotient, 3),
},
[23] = {
[ts_sym_expression] = SHIFT(24),
@ -487,23 +487,23 @@ PARSE_TABLE = {
[ts_sym_group] = SHIFT(25),
[ts_sym_number] = SHIFT(25),
[ts_sym_variable] = SHIFT(25),
[ts_aux_sym_token5] = SHIFT(26),
[ts_aux_sym_6] = SHIFT(26),
},
[24] = {
[ts_aux_sym_token0] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token1] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token2] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token3] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token4] = SHIFT(23),
[ts_aux_sym_token6] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_1] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_2] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_3] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_4] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_5] = SHIFT(23),
[ts_aux_sym_7] = REDUCE(ts_sym_exponent, 3),
},
[25] = {
[ts_aux_sym_token0] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token1] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token2] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token3] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token4] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token6] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_1] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_2] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_3] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_4] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_5] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_7] = REDUCE(ts_sym_expression, 1),
},
[26] = {
[ts_sym_expression] = SHIFT(27),
@ -516,37 +516,37 @@ PARSE_TABLE = {
[ts_builtin_sym_error] = SHIFT(29),
[ts_sym_number] = SHIFT(25),
[ts_sym_variable] = SHIFT(25),
[ts_aux_sym_token5] = SHIFT(26),
[ts_aux_sym_6] = SHIFT(26),
},
[27] = {
[ts_aux_sym_token0] = SHIFT(15),
[ts_aux_sym_token1] = SHIFT(17),
[ts_aux_sym_token2] = SHIFT(19),
[ts_aux_sym_token3] = SHIFT(21),
[ts_aux_sym_token4] = SHIFT(23),
[ts_aux_sym_token6] = SHIFT(28),
[ts_aux_sym_1] = SHIFT(15),
[ts_aux_sym_2] = SHIFT(17),
[ts_aux_sym_3] = SHIFT(19),
[ts_aux_sym_4] = SHIFT(21),
[ts_aux_sym_5] = SHIFT(23),
[ts_aux_sym_7] = SHIFT(28),
},
[28] = {
[ts_aux_sym_token0] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token1] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token2] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token3] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token4] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token6] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_1] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_2] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_3] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_4] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_5] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_7] = REDUCE(ts_sym_group, 3),
},
[29] = {
[ts_aux_sym_token6] = SHIFT(28),
[ts_aux_sym_7] = SHIFT(28),
},
[30] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token0] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token1] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token2] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token3] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token4] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_1] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_2] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_3] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_4] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_5] = REDUCE(ts_sym_group, 3),
},
[31] = {
[ts_aux_sym_token6] = SHIFT(30),
[ts_aux_sym_7] = SHIFT(30),
},
};

1252
examples/parsers/golang.c
View file

File diff suppressed because it is too large Load diff

32156
examples/parsers/javascript.c
View file

File diff suppressed because it is too large Load diff

224
examples/parsers/json.c
View file

@ -14,12 +14,12 @@ enum {
ts_sym_false,
ts_aux_sym_object_repeat0,
ts_aux_sym_array_repeat0,
ts_aux_sym_token0,
ts_aux_sym_token1,
ts_aux_sym_token2,
ts_aux_sym_token3,
ts_aux_sym_token4,
ts_aux_sym_token5,
ts_aux_sym_1,
ts_aux_sym_2,
ts_aux_sym_3,
ts_aux_sym_4,
ts_aux_sym_5,
ts_aux_sym_6,
};
SYMBOL_NAMES = {
@ -35,12 +35,12 @@ SYMBOL_NAMES = {
[ts_sym_false] = "false",
[ts_aux_sym_object_repeat0] = "object_repeat0",
[ts_aux_sym_array_repeat0] = "array_repeat0",
[ts_aux_sym_token0] = "",
[ts_aux_sym_token1] = "",
[ts_aux_sym_token2] = "",
[ts_aux_sym_token3] = "",
[ts_aux_sym_token4] = "",
[ts_aux_sym_token5] = "",
[ts_aux_sym_1] = "'{'",
[ts_aux_sym_2] = "':'",
[ts_aux_sym_3] = "','",
[ts_aux_sym_4] = "'}'",
[ts_aux_sym_5] = "'['",
[ts_aux_sym_6] = "']'",
};
UBIQUITOUS_SYMBOLS = {
@ -49,12 +49,12 @@ UBIQUITOUS_SYMBOLS = {
HIDDEN_SYMBOLS = {
[ts_aux_sym_object_repeat0] = 1,
[ts_aux_sym_array_repeat0] = 1,
[ts_aux_sym_token0] = 1,
[ts_aux_sym_token1] = 1,
[ts_aux_sym_token2] = 1,
[ts_aux_sym_token3] = 1,
[ts_aux_sym_token4] = 1,
[ts_aux_sym_token5] = 1,
[ts_aux_sym_1] = 1,
[ts_aux_sym_2] = 1,
[ts_aux_sym_3] = 1,
[ts_aux_sym_4] = 1,
[ts_aux_sym_5] = 1,
[ts_aux_sym_6] = 1,
};
LEX_FN() {
@ -125,7 +125,7 @@ LEX_FN() {
ADVANCE(8);
ACCEPT_TOKEN(ts_sym_number);
case 9:
ACCEPT_TOKEN(ts_aux_sym_token4);
ACCEPT_TOKEN(ts_aux_sym_5);
case 10:
if (lookahead == 'a')
ADVANCE(11);
@ -173,7 +173,7 @@ LEX_FN() {
case 22:
ACCEPT_TOKEN(ts_sym_true);
case 23:
ACCEPT_TOKEN(ts_aux_sym_token0);
ACCEPT_TOKEN(ts_aux_sym_1);
case 24:
START_TOKEN();
if (lookahead == '\0')
@ -198,7 +198,7 @@ LEX_FN() {
ADVANCE(27);
LEX_ERROR();
case 27:
ACCEPT_TOKEN(ts_aux_sym_token3);
ACCEPT_TOKEN(ts_aux_sym_4);
case 28:
START_TOKEN();
if (('\t' <= lookahead && lookahead <= '\n') ||
@ -211,7 +211,7 @@ LEX_FN() {
ADVANCE(27);
LEX_ERROR();
case 29:
ACCEPT_TOKEN(ts_aux_sym_token2);
ACCEPT_TOKEN(ts_aux_sym_3);
case 30:
START_TOKEN();
if ((lookahead == '\t') ||
@ -243,7 +243,7 @@ LEX_FN() {
ADVANCE(33);
LEX_ERROR();
case 33:
ACCEPT_TOKEN(ts_aux_sym_token1);
ACCEPT_TOKEN(ts_aux_sym_2);
case 34:
START_TOKEN();
if (('\t' <= lookahead && lookahead <= '\n') ||
@ -268,7 +268,7 @@ LEX_FN() {
ADVANCE(23);
LEX_ERROR();
case 35:
ACCEPT_TOKEN(ts_aux_sym_token5);
ACCEPT_TOKEN(ts_aux_sym_6);
case 36:
START_TOKEN();
if (('\t' <= lookahead && lookahead <= '\n') ||
@ -433,8 +433,8 @@ PARSE_TABLE = {
[ts_sym_null] = SHIFT(2),
[ts_sym_true] = SHIFT(2),
[ts_sym_false] = SHIFT(2),
[ts_aux_sym_token0] = SHIFT(3),
[ts_aux_sym_token4] = SHIFT(55),
[ts_aux_sym_1] = SHIFT(3),
[ts_aux_sym_5] = SHIFT(55),
},
[1] = {
[ts_builtin_sym_end] = ACCEPT_INPUT(),
@ -445,15 +445,15 @@ PARSE_TABLE = {
[3] = {
[ts_builtin_sym_error] = SHIFT(4),
[ts_sym_string] = SHIFT(49),
[ts_aux_sym_token3] = SHIFT(54),
[ts_aux_sym_4] = SHIFT(54),
},
[4] = {
[ts_aux_sym_object_repeat0] = SHIFT(5),
[ts_aux_sym_token2] = SHIFT(7),
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 0),
[ts_aux_sym_3] = SHIFT(7),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 0),
},
[5] = {
[ts_aux_sym_token3] = SHIFT(6),
[ts_aux_sym_4] = SHIFT(6),
},
[6] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_object, 4),
@ -464,14 +464,14 @@ PARSE_TABLE = {
},
[8] = {
[ts_aux_sym_object_repeat0] = SHIFT(9),
[ts_aux_sym_token2] = SHIFT(7),
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 0),
[ts_aux_sym_3] = SHIFT(7),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 0),
},
[9] = {
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 3),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 3),
},
[10] = {
[ts_aux_sym_token1] = SHIFT(11),
[ts_aux_sym_2] = SHIFT(11),
},
[11] = {
[ts_sym_value] = SHIFT(12),
@ -482,40 +482,40 @@ PARSE_TABLE = {
[ts_sym_null] = SHIFT(14),
[ts_sym_true] = SHIFT(14),
[ts_sym_false] = SHIFT(14),
[ts_aux_sym_token0] = SHIFT(15),
[ts_aux_sym_token4] = SHIFT(24),
[ts_aux_sym_1] = SHIFT(15),
[ts_aux_sym_5] = SHIFT(24),
},
[12] = {
[ts_aux_sym_object_repeat0] = SHIFT(13),
[ts_aux_sym_token2] = SHIFT(7),
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 0),
[ts_aux_sym_3] = SHIFT(7),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 0),
},
[13] = {
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 5),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 5),
},
[14] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_value, 1),
[ts_aux_sym_token3] = REDUCE(ts_sym_value, 1),
[ts_aux_sym_3] = REDUCE(ts_sym_value, 1),
[ts_aux_sym_4] = REDUCE(ts_sym_value, 1),
},
[15] = {
[ts_builtin_sym_error] = SHIFT(16),
[ts_sym_string] = SHIFT(19),
[ts_aux_sym_token3] = SHIFT(48),
[ts_aux_sym_4] = SHIFT(48),
},
[16] = {
[ts_aux_sym_object_repeat0] = SHIFT(17),
[ts_aux_sym_token2] = SHIFT(7),
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 0),
[ts_aux_sym_3] = SHIFT(7),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 0),
},
[17] = {
[ts_aux_sym_token3] = SHIFT(18),
[ts_aux_sym_4] = SHIFT(18),
},
[18] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_object, 4),
[ts_aux_sym_token3] = REDUCE(ts_sym_object, 4),
[ts_aux_sym_3] = REDUCE(ts_sym_object, 4),
[ts_aux_sym_4] = REDUCE(ts_sym_object, 4),
},
[19] = {
[ts_aux_sym_token1] = SHIFT(20),
[ts_aux_sym_2] = SHIFT(20),
},
[20] = {
[ts_sym_value] = SHIFT(21),
@ -526,20 +526,20 @@ PARSE_TABLE = {
[ts_sym_null] = SHIFT(14),
[ts_sym_true] = SHIFT(14),
[ts_sym_false] = SHIFT(14),
[ts_aux_sym_token0] = SHIFT(15),
[ts_aux_sym_token4] = SHIFT(24),
[ts_aux_sym_1] = SHIFT(15),
[ts_aux_sym_5] = SHIFT(24),
},
[21] = {
[ts_aux_sym_object_repeat0] = SHIFT(22),
[ts_aux_sym_token2] = SHIFT(7),
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 0),
[ts_aux_sym_3] = SHIFT(7),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 0),
},
[22] = {
[ts_aux_sym_token3] = SHIFT(23),
[ts_aux_sym_4] = SHIFT(23),
},
[23] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_object, 6),
[ts_aux_sym_token3] = REDUCE(ts_sym_object, 6),
[ts_aux_sym_3] = REDUCE(ts_sym_object, 6),
[ts_aux_sym_4] = REDUCE(ts_sym_object, 6),
},
[24] = {
[ts_sym_value] = SHIFT(25),
@ -551,21 +551,21 @@ PARSE_TABLE = {
[ts_sym_null] = SHIFT(31),
[ts_sym_true] = SHIFT(31),
[ts_sym_false] = SHIFT(31),
[ts_aux_sym_token0] = SHIFT(32),
[ts_aux_sym_token4] = SHIFT(42),
[ts_aux_sym_token5] = SHIFT(47),
[ts_aux_sym_1] = SHIFT(32),
[ts_aux_sym_5] = SHIFT(42),
[ts_aux_sym_6] = SHIFT(47),
},
[25] = {
[ts_aux_sym_array_repeat0] = SHIFT(26),
[ts_aux_sym_token2] = SHIFT(28),
[ts_aux_sym_token5] = REDUCE(ts_aux_sym_array_repeat0, 0),
[ts_aux_sym_3] = SHIFT(28),
[ts_aux_sym_6] = REDUCE(ts_aux_sym_array_repeat0, 0),
},
[26] = {
[ts_aux_sym_token5] = SHIFT(27),
[ts_aux_sym_6] = SHIFT(27),
},
[27] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_array, 4),
[ts_aux_sym_token3] = REDUCE(ts_sym_array, 4),
[ts_aux_sym_3] = REDUCE(ts_sym_array, 4),
[ts_aux_sym_4] = REDUCE(ts_sym_array, 4),
},
[28] = {
[ts_sym_value] = SHIFT(29),
@ -577,40 +577,40 @@ PARSE_TABLE = {
[ts_sym_null] = SHIFT(31),
[ts_sym_true] = SHIFT(31),
[ts_sym_false] = SHIFT(31),
[ts_aux_sym_token0] = SHIFT(32),
[ts_aux_sym_token4] = SHIFT(42),
[ts_aux_sym_1] = SHIFT(32),
[ts_aux_sym_5] = SHIFT(42),
},
[29] = {
[ts_aux_sym_array_repeat0] = SHIFT(30),
[ts_aux_sym_token2] = SHIFT(28),
[ts_aux_sym_token5] = REDUCE(ts_aux_sym_array_repeat0, 0),
[ts_aux_sym_3] = SHIFT(28),
[ts_aux_sym_6] = REDUCE(ts_aux_sym_array_repeat0, 0),
},
[30] = {
[ts_aux_sym_token5] = REDUCE(ts_aux_sym_array_repeat0, 3),
[ts_aux_sym_6] = REDUCE(ts_aux_sym_array_repeat0, 3),
},
[31] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_value, 1),
[ts_aux_sym_token5] = REDUCE(ts_sym_value, 1),
[ts_aux_sym_3] = REDUCE(ts_sym_value, 1),
[ts_aux_sym_6] = REDUCE(ts_sym_value, 1),
},
[32] = {
[ts_builtin_sym_error] = SHIFT(33),
[ts_sym_string] = SHIFT(36),
[ts_aux_sym_token3] = SHIFT(41),
[ts_aux_sym_4] = SHIFT(41),
},
[33] = {
[ts_aux_sym_object_repeat0] = SHIFT(34),
[ts_aux_sym_token2] = SHIFT(7),
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 0),
[ts_aux_sym_3] = SHIFT(7),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 0),
},
[34] = {
[ts_aux_sym_token3] = SHIFT(35),
[ts_aux_sym_4] = SHIFT(35),
},
[35] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_object, 4),
[ts_aux_sym_token5] = REDUCE(ts_sym_object, 4),
[ts_aux_sym_3] = REDUCE(ts_sym_object, 4),
[ts_aux_sym_6] = REDUCE(ts_sym_object, 4),
},
[36] = {
[ts_aux_sym_token1] = SHIFT(37),
[ts_aux_sym_2] = SHIFT(37),
},
[37] = {
[ts_sym_value] = SHIFT(38),
@ -621,24 +621,24 @@ PARSE_TABLE = {
[ts_sym_null] = SHIFT(14),
[ts_sym_true] = SHIFT(14),
[ts_sym_false] = SHIFT(14),
[ts_aux_sym_token0] = SHIFT(15),
[ts_aux_sym_token4] = SHIFT(24),
[ts_aux_sym_1] = SHIFT(15),
[ts_aux_sym_5] = SHIFT(24),
},
[38] = {
[ts_aux_sym_object_repeat0] = SHIFT(39),
[ts_aux_sym_token2] = SHIFT(7),
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 0),
[ts_aux_sym_3] = SHIFT(7),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 0),
},
[39] = {
[ts_aux_sym_token3] = SHIFT(40),
[ts_aux_sym_4] = SHIFT(40),
},
[40] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_object, 6),
[ts_aux_sym_token5] = REDUCE(ts_sym_object, 6),
[ts_aux_sym_3] = REDUCE(ts_sym_object, 6),
[ts_aux_sym_6] = REDUCE(ts_sym_object, 6),
},
[41] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_object, 2),
[ts_aux_sym_token5] = REDUCE(ts_sym_object, 2),
[ts_aux_sym_3] = REDUCE(ts_sym_object, 2),
[ts_aux_sym_6] = REDUCE(ts_sym_object, 2),
},
[42] = {
[ts_sym_value] = SHIFT(43),
@ -650,36 +650,36 @@ PARSE_TABLE = {
[ts_sym_null] = SHIFT(31),
[ts_sym_true] = SHIFT(31),
[ts_sym_false] = SHIFT(31),
[ts_aux_sym_token0] = SHIFT(32),
[ts_aux_sym_token4] = SHIFT(42),
[ts_aux_sym_token5] = SHIFT(46),
[ts_aux_sym_1] = SHIFT(32),
[ts_aux_sym_5] = SHIFT(42),
[ts_aux_sym_6] = SHIFT(46),
},
[43] = {
[ts_aux_sym_array_repeat0] = SHIFT(44),
[ts_aux_sym_token2] = SHIFT(28),
[ts_aux_sym_token5] = REDUCE(ts_aux_sym_array_repeat0, 0),
[ts_aux_sym_3] = SHIFT(28),
[ts_aux_sym_6] = REDUCE(ts_aux_sym_array_repeat0, 0),
},
[44] = {
[ts_aux_sym_token5] = SHIFT(45),
[ts_aux_sym_6] = SHIFT(45),
},
[45] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_array, 4),
[ts_aux_sym_token5] = REDUCE(ts_sym_array, 4),
[ts_aux_sym_3] = REDUCE(ts_sym_array, 4),
[ts_aux_sym_6] = REDUCE(ts_sym_array, 4),
},
[46] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_array, 2),
[ts_aux_sym_token5] = REDUCE(ts_sym_array, 2),
[ts_aux_sym_3] = REDUCE(ts_sym_array, 2),
[ts_aux_sym_6] = REDUCE(ts_sym_array, 2),
},
[47] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_array, 2),
[ts_aux_sym_token3] = REDUCE(ts_sym_array, 2),
[ts_aux_sym_3] = REDUCE(ts_sym_array, 2),
[ts_aux_sym_4] = REDUCE(ts_sym_array, 2),
},
[48] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_object, 2),
[ts_aux_sym_token3] = REDUCE(ts_sym_object, 2),
[ts_aux_sym_3] = REDUCE(ts_sym_object, 2),
[ts_aux_sym_4] = REDUCE(ts_sym_object, 2),
},
[49] = {
[ts_aux_sym_token1] = SHIFT(50),
[ts_aux_sym_2] = SHIFT(50),
},
[50] = {
[ts_sym_value] = SHIFT(51),
@ -690,16 +690,16 @@ PARSE_TABLE = {
[ts_sym_null] = SHIFT(14),
[ts_sym_true] = SHIFT(14),
[ts_sym_false] = SHIFT(14),
[ts_aux_sym_token0] = SHIFT(15),
[ts_aux_sym_token4] = SHIFT(24),
[ts_aux_sym_1] = SHIFT(15),
[ts_aux_sym_5] = SHIFT(24),
},
[51] = {
[ts_aux_sym_object_repeat0] = SHIFT(52),
[ts_aux_sym_token2] = SHIFT(7),
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 0),
[ts_aux_sym_3] = SHIFT(7),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 0),
},
[52] = {
[ts_aux_sym_token3] = SHIFT(53),
[ts_aux_sym_4] = SHIFT(53),
},
[53] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_object, 6),
@ -717,17 +717,17 @@ PARSE_TABLE = {
[ts_sym_null] = SHIFT(31),
[ts_sym_true] = SHIFT(31),
[ts_sym_false] = SHIFT(31),
[ts_aux_sym_token0] = SHIFT(32),
[ts_aux_sym_token4] = SHIFT(42),
[ts_aux_sym_token5] = SHIFT(59),
[ts_aux_sym_1] = SHIFT(32),
[ts_aux_sym_5] = SHIFT(42),
[ts_aux_sym_6] = SHIFT(59),
},
[56] = {
[ts_aux_sym_array_repeat0] = SHIFT(57),
[ts_aux_sym_token2] = SHIFT(28),
[ts_aux_sym_token5] = REDUCE(ts_aux_sym_array_repeat0, 0),
[ts_aux_sym_3] = SHIFT(28),
[ts_aux_sym_6] = REDUCE(ts_aux_sym_array_repeat0, 0),
},
[57] = {
[ts_aux_sym_token5] = SHIFT(58),
[ts_aux_sym_6] = SHIFT(58),
},
[58] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_array, 4),

12
spec/compiler/prepare_grammar/extract_tokens_spec.cc
View file

@ -18,7 +18,7 @@ describe("extracting tokens from a grammar", []() {
}, {})));
AssertThat(result.second, Equals(PreparedGrammar({}, {
{ "token0", str("ab") },
{ "'ab'", str("ab") },
})));
});
@ -32,14 +32,14 @@ describe("extracting tokens from a grammar", []() {
}, {})));
AssertThat(result.second, Equals(PreparedGrammar({}, {
{ "token0", pattern("a+") },
{ "/a+/", pattern("a+") },
})));
});
it("moves other rules marked as tokens into the lexical grammar", [&]() {
pair<PreparedGrammar, PreparedGrammar> result = extract_tokens(PreparedGrammar({
{ "rule_A", seq({
token(choice({ str("a"), str("b") })),
token(seq({ pattern("."), choice({ str("a"), str("b") }) })),
i_sym(0) }) }
}, {}));
@ -48,7 +48,7 @@ describe("extracting tokens from a grammar", []() {
}, {})));
AssertThat(result.second, Equals(PreparedGrammar({}, {
{ "token0", token(choice({ str("a"), str("b") })) },
{ "(seq /./ (choice 'a' 'b'))", token(seq({ pattern("."), choice({ str("a"), str("b") }) })) },
})));
});
@ -74,7 +74,7 @@ describe("extracting tokens from a grammar", []() {
}, {})));
AssertThat(result.second, Equals(PreparedGrammar({}, {
{ "token0", str("ab") },
{ "'ab'", str("ab") },
})));
});
@ -88,7 +88,7 @@ describe("extracting tokens from a grammar", []() {
})));
AssertThat(result.second, Equals(PreparedGrammar({}, {
{ "token0", str("ab") },
{ "'ab'", str("ab") },
})));
});

40
src/compiler/generate_code/c_code.cc
View file

@ -7,7 +7,6 @@
#include "compiler/util/string_helpers.h"
#include "compiler/rules/built_in_symbols.h"
#include "compiler/prepared_grammar.h"
#include "compiler/generate_code/token_description.h"
namespace tree_sitter {
using std::string;
@ -56,6 +55,7 @@ namespace tree_sitter {
const LexTable lex_table;
const PreparedGrammar syntax_grammar;
const PreparedGrammar lexical_grammar;
map<string, string> sanitized_names;
public:
CCodeGenerator(string name,
@ -88,6 +88,40 @@ namespace tree_sitter {
const PreparedGrammar & grammar_for_symbol(const rules::Symbol &symbol) {
return symbol.is_token() ? lexical_grammar : syntax_grammar;
}
string sanitize_name(string name) {
auto existing = sanitized_names.find(name);
if (existing != sanitized_names.end())
return existing->second;
string stripped_name;
for (char c : name) {
if (('a' <= c && c <= 'z') ||
('A' <= c && c <= 'Z') ||
('0' <= c && c <= '9') ||
(c == '_')) {
stripped_name += c;
}
}
for (size_t extra_number = 0;; extra_number++) {
string suffix = extra_number ? to_string(extra_number) : "";
string unique_name = stripped_name + suffix;
if (unique_name == "")
continue;
if (!has_sanitized_name(unique_name)) {
sanitized_names.insert({ name, unique_name });
return unique_name;
}
}
}
bool has_sanitized_name(string name) {
for (auto &pair : sanitized_names)
if (pair.second == name)
return true;
return false;
}
string symbol_id(const rules::Symbol &symbol) {
if (symbol.is_built_in()) {
@ -95,7 +129,7 @@ namespace tree_sitter {
"ts_builtin_sym_error" :
"ts_builtin_sym_end";
} else {
string name = grammar_for_symbol(symbol).rule_name(symbol);
string name = sanitize_name(grammar_for_symbol(symbol).rule_name(symbol));
if (symbol.is_auxiliary())
return "ts_aux_sym_" + name;
else
@ -111,7 +145,7 @@ namespace tree_sitter {
if (symbol.is_built_in()) {
return (symbol == rules::ERROR()) ? "error" : "end";
} else if (symbol.is_token() && symbol.is_auxiliary()) {
return token_description(grammar_for_symbol(symbol).rule(symbol));
return grammar_for_symbol(symbol).rule_name(symbol);
} else {
return grammar_for_symbol(symbol).rule_name(symbol);
}

7
src/compiler/prepare_grammar/extract_tokens.cc
View file

@ -9,6 +9,7 @@
#include "compiler/rules/string.h"
#include "compiler/rules/metadata.h"
#include "compiler/rules/pattern.h"
#include "compiler/prepare_grammar/token_description.h"
namespace tree_sitter {
using std::pair;
@ -60,13 +61,13 @@ namespace tree_sitter {
const rules::SymbolOption SymbolOptionAuxToken = rules::SymbolOption(rules::SymbolOptionToken|rules::SymbolOptionAuxiliary);
class TokenExtractor : public rules::IdentityRuleFn {
rule_ptr apply_to_token(const rules::Rule *rule) {
auto result = rule->copy();
rule_ptr apply_to_token(const rules::Rule *input) {
auto rule = input->copy();
for (size_t i = 0; i < tokens.size(); i++)
if (tokens[i].second->operator==(*rule))
return make_shared<Symbol>(i, SymbolOptionAuxToken);
size_t index = tokens.size();
tokens.push_back({ "token" + to_string(index), result });
tokens.push_back({ token_description(rule), rule });
return make_shared<Symbol>(index, SymbolOptionAuxToken);
}

2
src/compiler/prepare_grammar/prepare_grammar.cc
View file

@ -5,8 +5,6 @@
#include "compiler/prepare_grammar/expand_tokens.h"
#include "compiler/prepare_grammar/intern_symbols.h"
#include "stream_methods.h"
namespace tree_sitter {
using std::tuple;
using std::make_tuple;

19
src/compiler/generate_code/token_description.cc → src/compiler/prepare_grammar/token_description.cc
View file

@ -1,14 +1,18 @@
#include "compiler/generate_code/token_description.h"
#include "compiler/prepare_grammar/token_description.h"
#include "tree_sitter/compiler.h"
#include "compiler/rules/visitor.h"
#include "compiler/rules/pattern.h"
#include "compiler/rules/seq.h"
#include "compiler/rules/choice.h"
#include "compiler/rules/seq.h"
#include "compiler/rules/string.h"
#include "compiler/rules/metadata.h"
#include "compiler/util/string_helpers.h"
namespace tree_sitter {
using std::string;
namespace generate_code {
namespace prepare_grammar {
class TokenDescription : public rules::RuleFn<string> {
string apply_to(const rules::Pattern *rule) {
return "/" + rule->value + "/";
@ -21,6 +25,17 @@ namespace tree_sitter {
string apply_to(const rules::Metadata *rule) {
return apply(rule->rule);
}
string apply_to(const rules::Seq *rule) {
return "(seq " + apply(rule->left) + " " + apply(rule->right) + ")";
}
string apply_to(const rules::Choice *rule) {
string result = "(choice";
for (auto &element : rule->elements)
result += " " + apply(element);
return result + ")";
}
};
std::string token_description(const rules::rule_ptr &rule) {

2
src/compiler/generate_code/token_description.h → src/compiler/prepare_grammar/token_description.h
View file

@ -5,7 +5,7 @@
#include "tree_sitter/compiler.h"
namespace tree_sitter {
namespace generate_code {
namespace prepare_grammar {
std::string token_description(const rules::rule_ptr &);
}
}