traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Intern symbols during grammar preparation

Browse source
This commit is contained in:
Max Brunsfeld 2014-04-22 23:38:26 -07:00
parent 33d781f492
commit 68d44fd565
67 changed files with 10829 additions and 10557 deletions
Download patch file Download diff file Expand all files Collapse all files

563
examples/parsers/arithmetic.c
View file

@ -4,53 +4,53 @@
#define SYMBOL_COUNT 18
enum {
ts_sym_difference = 2,
ts_sym_exponent = 3,
ts_sym_expression = 4,
ts_sym_group = 5,
ts_sym_number = 6,
ts_sym_product = 7,
ts_sym_quotient = 8,
ts_sym_sum = 9,
ts_sym_expression = 2,
ts_sym_sum = 3,
ts_sym_difference = 4,
ts_sym_product = 5,
ts_sym_quotient = 6,
ts_sym_exponent = 7,
ts_sym_group = 8,
ts_sym_number = 9,
ts_sym_variable = 10,
ts_aux_sym_token1 = 11,
ts_aux_sym_token2 = 12,
ts_aux_sym_token3 = 13,
ts_aux_sym_token4 = 14,
ts_aux_sym_token5 = 15,
ts_aux_sym_token6 = 16,
ts_aux_sym_token7 = 17,
ts_aux_sym_token0 = 11,
ts_aux_sym_token1 = 12,
ts_aux_sym_token2 = 13,
ts_aux_sym_token3 = 14,
ts_aux_sym_token4 = 15,
ts_aux_sym_token5 = 16,
ts_aux_sym_token6 = 17,
};
SYMBOL_NAMES = {
[ts_sym_difference] = "difference",
[ts_sym_exponent] = "exponent",
[ts_sym_expression] = "expression",
[ts_sym_group] = "group",
[ts_sym_number] = "number",
[ts_sym_sum] = "sum",
[ts_sym_difference] = "difference",
[ts_sym_product] = "product",
[ts_sym_quotient] = "quotient",
[ts_sym_sum] = "sum",
[ts_sym_exponent] = "exponent",
[ts_sym_group] = "group",
[ts_builtin_sym_error] = "error",
[ts_builtin_sym_end] = "end",
[ts_sym_number] = "number",
[ts_sym_variable] = "variable",
[ts_aux_sym_token1] = "'+'",
[ts_aux_sym_token2] = "'-'",
[ts_aux_sym_token3] = "'*'",
[ts_aux_sym_token4] = "'/'",
[ts_aux_sym_token5] = "'^'",
[ts_aux_sym_token6] = "'('",
[ts_aux_sym_token7] = "')'",
[ts_builtin_sym_end] = "EOF",
[ts_builtin_sym_error] = "ERROR",
[ts_aux_sym_token0] = "'+'",
[ts_aux_sym_token1] = "'-'",
[ts_aux_sym_token2] = "'*'",
[ts_aux_sym_token3] = "'/'",
[ts_aux_sym_token4] = "'^'",
[ts_aux_sym_token5] = "'('",
[ts_aux_sym_token6] = "')'",
};
HIDDEN_SYMBOLS = {
[ts_aux_sym_token0] = 1,
[ts_aux_sym_token1] = 1,
[ts_aux_sym_token2] = 1,
[ts_aux_sym_token3] = 1,
[ts_aux_sym_token4] = 1,
[ts_aux_sym_token5] = 1,
[ts_aux_sym_token6] = 1,
[ts_aux_sym_token7] = 1,
};
LEX_FN() {
@ -78,15 +78,15 @@ LEX_FN() {
case 1:
ACCEPT_TOKEN(ts_builtin_sym_end);
case 2:
ACCEPT_TOKEN(ts_aux_sym_token3);
case 3:
ACCEPT_TOKEN(ts_aux_sym_token1);
case 4:
ACCEPT_TOKEN(ts_aux_sym_token2);
case 3:
ACCEPT_TOKEN(ts_aux_sym_token0);
case 4:
ACCEPT_TOKEN(ts_aux_sym_token1);
case 5:
ACCEPT_TOKEN(ts_aux_sym_token4);
ACCEPT_TOKEN(ts_aux_sym_token3);
case 6:
ACCEPT_TOKEN(ts_aux_sym_token5);
ACCEPT_TOKEN(ts_aux_sym_token4);
case 7:
START_TOKEN();
if (('\t' <= lookahead && lookahead <= '\n') ||
@ -107,7 +107,7 @@ LEX_FN() {
ADVANCE(6);
LEX_ERROR();
case 8:
ACCEPT_TOKEN(ts_aux_sym_token7);
ACCEPT_TOKEN(ts_aux_sym_token6);
case 9:
START_TOKEN();
if ((lookahead == '\t') ||
@ -133,7 +133,7 @@ LEX_FN() {
ADVANCE(13);
LEX_ERROR();
case 11:
ACCEPT_TOKEN(ts_aux_sym_token6);
ACCEPT_TOKEN(ts_aux_sym_token5);
case 12:
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(12);
@ -174,7 +174,6 @@ LEX_FN() {
ADVANCE(6);
LEX_ERROR();
case ts_lex_state_error:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
@ -209,29 +208,29 @@ LEX_FN() {
LEX_STATES = {
[0] = 10,
[1] = 0,
[2] = 0,
[3] = 10,
[4] = 0,
[5] = 10,
[6] = 0,
[7] = 10,
[8] = 0,
[9] = 10,
[10] = 0,
[11] = 10,
[2] = 10,
[3] = 0,
[4] = 10,
[5] = 0,
[6] = 10,
[7] = 0,
[8] = 10,
[9] = 0,
[10] = 10,
[11] = 0,
[12] = 0,
[13] = 10,
[14] = 7,
[15] = 7,
[16] = 10,
[17] = 7,
[18] = 10,
[19] = 7,
[20] = 10,
[21] = 7,
[22] = 10,
[23] = 7,
[24] = 10,
[15] = 10,
[16] = 7,
[17] = 10,
[18] = 7,
[19] = 10,
[20] = 7,
[21] = 10,
[22] = 7,
[23] = 10,
[24] = 7,
[25] = 7,
[26] = 10,
[27] = 7,
@ -246,304 +245,304 @@ LEX_STATES = {
PARSE_TABLE = {
[0] = {
[ts_sym_difference] = SHIFT(1),
[ts_sym_exponent] = SHIFT(1),
[ts_sym_expression] = SHIFT(2),
[ts_sym_group] = SHIFT(1),
[ts_sym_number] = SHIFT(1),
[ts_sym_product] = SHIFT(1),
[ts_sym_quotient] = SHIFT(1),
[ts_sym_sum] = SHIFT(1),
[ts_sym_variable] = SHIFT(1),
[ts_aux_sym_token6] = SHIFT(13),
[ts_sym_expression] = SHIFT(1),
[ts_sym_sum] = SHIFT(12),
[ts_sym_difference] = SHIFT(12),
[ts_sym_product] = SHIFT(12),
[ts_sym_quotient] = SHIFT(12),
[ts_sym_exponent] = SHIFT(12),
[ts_sym_group] = SHIFT(12),
[ts_sym_number] = SHIFT(12),
[ts_sym_variable] = SHIFT(12),
[ts_aux_sym_token5] = SHIFT(13),
},
[1] = {
[ts_aux_sym_token1] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token2] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token3] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token4] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token5] = REDUCE(ts_sym_expression, 1),
[ts_builtin_sym_end] = REDUCE(ts_sym_expression, 1),
[ts_builtin_sym_end] = ACCEPT_INPUT(),
[ts_aux_sym_token0] = SHIFT(2),
[ts_aux_sym_token1] = SHIFT(4),
[ts_aux_sym_token2] = SHIFT(6),
[ts_aux_sym_token3] = SHIFT(8),
[ts_aux_sym_token4] = SHIFT(10),
},
[2] = {
[ts_aux_sym_token1] = SHIFT(3),
[ts_aux_sym_token2] = SHIFT(5),
[ts_aux_sym_token3] = SHIFT(7),
[ts_aux_sym_token4] = SHIFT(9),
[ts_aux_sym_token5] = SHIFT(11),
[ts_builtin_sym_end] = ACCEPT_INPUT(),
[ts_sym_expression] = SHIFT(3),
[ts_sym_sum] = SHIFT(12),
[ts_sym_difference] = SHIFT(12),
[ts_sym_product] = SHIFT(12),
[ts_sym_quotient] = SHIFT(12),
[ts_sym_exponent] = SHIFT(12),
[ts_sym_group] = SHIFT(12),
[ts_sym_number] = SHIFT(12),
[ts_sym_variable] = SHIFT(12),
[ts_aux_sym_token5] = SHIFT(13),
},
[3] = {
[ts_sym_difference] = SHIFT(1),
[ts_sym_exponent] = SHIFT(1),
[ts_sym_expression] = SHIFT(4),
[ts_sym_group] = SHIFT(1),
[ts_sym_number] = SHIFT(1),
[ts_sym_product] = SHIFT(1),
[ts_sym_quotient] = SHIFT(1),
[ts_sym_sum] = SHIFT(1),
[ts_sym_variable] = SHIFT(1),
[ts_aux_sym_token6] = SHIFT(13),
[ts_builtin_sym_end] = REDUCE(ts_sym_sum, 3),
[ts_aux_sym_token0] = SHIFT(2),
[ts_aux_sym_token1] = SHIFT(4),
[ts_aux_sym_token2] = SHIFT(6),
[ts_aux_sym_token3] = SHIFT(8),
[ts_aux_sym_token4] = SHIFT(10),
},
[4] = {
[ts_aux_sym_token1] = SHIFT(3),
[ts_aux_sym_token2] = SHIFT(5),
[ts_aux_sym_token3] = SHIFT(7),
[ts_aux_sym_token4] = SHIFT(9),
[ts_aux_sym_token5] = SHIFT(11),
[ts_builtin_sym_end] = REDUCE(ts_sym_sum, 3),
[ts_sym_expression] = SHIFT(5),
[ts_sym_sum] = SHIFT(12),
[ts_sym_difference] = SHIFT(12),
[ts_sym_product] = SHIFT(12),
[ts_sym_quotient] = SHIFT(12),
[ts_sym_exponent] = SHIFT(12),
[ts_sym_group] = SHIFT(12),
[ts_sym_number] = SHIFT(12),
[ts_sym_variable] = SHIFT(12),
[ts_aux_sym_token5] = SHIFT(13),
},
[5] = {
[ts_sym_difference] = SHIFT(1),
[ts_sym_exponent] = SHIFT(1),
[ts_sym_expression] = SHIFT(6),
[ts_sym_group] = SHIFT(1),
[ts_sym_number] = SHIFT(1),
[ts_sym_product] = SHIFT(1),
[ts_sym_quotient] = SHIFT(1),
[ts_sym_sum] = SHIFT(1),
[ts_sym_variable] = SHIFT(1),
[ts_aux_sym_token6] = SHIFT(13),
[ts_builtin_sym_end] = REDUCE(ts_sym_difference, 3),
[ts_aux_sym_token0] = SHIFT(2),
[ts_aux_sym_token1] = SHIFT(4),
[ts_aux_sym_token2] = SHIFT(6),
[ts_aux_sym_token3] = SHIFT(8),
[ts_aux_sym_token4] = SHIFT(10),
},
[6] = {
[ts_aux_sym_token1] = SHIFT(3),
[ts_aux_sym_token2] = SHIFT(5),
[ts_aux_sym_token3] = SHIFT(7),
[ts_aux_sym_token4] = SHIFT(9),
[ts_aux_sym_token5] = SHIFT(11),
[ts_builtin_sym_end] = REDUCE(ts_sym_difference, 3),
[ts_sym_expression] = SHIFT(7),
[ts_sym_sum] = SHIFT(12),
[ts_sym_difference] = SHIFT(12),
[ts_sym_product] = SHIFT(12),
[ts_sym_quotient] = SHIFT(12),
[ts_sym_exponent] = SHIFT(12),
[ts_sym_group] = SHIFT(12),
[ts_sym_number] = SHIFT(12),
[ts_sym_variable] = SHIFT(12),
[ts_aux_sym_token5] = SHIFT(13),
},
[7] = {
[ts_sym_difference] = SHIFT(1),
[ts_sym_exponent] = SHIFT(1),
[ts_sym_expression] = SHIFT(8),
[ts_sym_group] = SHIFT(1),
[ts_sym_number] = SHIFT(1),
[ts_sym_product] = SHIFT(1),
[ts_sym_quotient] = SHIFT(1),
[ts_sym_sum] = SHIFT(1),
[ts_sym_variable] = SHIFT(1),
[ts_aux_sym_token6] = SHIFT(13),
[ts_builtin_sym_end] = REDUCE(ts_sym_product, 3),
[ts_aux_sym_token0] = REDUCE(ts_sym_product, 3),
[ts_aux_sym_token1] = REDUCE(ts_sym_product, 3),
[ts_aux_sym_token2] = SHIFT(6),
[ts_aux_sym_token3] = SHIFT(8),
[ts_aux_sym_token4] = SHIFT(10),
},
[8] = {
[ts_aux_sym_token1] = REDUCE(ts_sym_product, 3),
[ts_aux_sym_token2] = REDUCE(ts_sym_product, 3),
[ts_aux_sym_token3] = SHIFT(7),
[ts_aux_sym_token4] = SHIFT(9),
[ts_aux_sym_token5] = SHIFT(11),
[ts_builtin_sym_end] = REDUCE(ts_sym_product, 3),
[ts_sym_expression] = SHIFT(9),
[ts_sym_sum] = SHIFT(12),
[ts_sym_difference] = SHIFT(12),
[ts_sym_product] = SHIFT(12),
[ts_sym_quotient] = SHIFT(12),
[ts_sym_exponent] = SHIFT(12),
[ts_sym_group] = SHIFT(12),
[ts_sym_number] = SHIFT(12),
[ts_sym_variable] = SHIFT(12),
[ts_aux_sym_token5] = SHIFT(13),
},
[9] = {
[ts_sym_difference] = SHIFT(1),
[ts_sym_exponent] = SHIFT(1),
[ts_sym_expression] = SHIFT(10),
[ts_sym_group] = SHIFT(1),
[ts_sym_number] = SHIFT(1),
[ts_sym_product] = SHIFT(1),
[ts_sym_quotient] = SHIFT(1),
[ts_sym_sum] = SHIFT(1),
[ts_sym_variable] = SHIFT(1),
[ts_aux_sym_token6] = SHIFT(13),
[ts_builtin_sym_end] = REDUCE(ts_sym_quotient, 3),
[ts_aux_sym_token0] = REDUCE(ts_sym_quotient, 3),
[ts_aux_sym_token1] = REDUCE(ts_sym_quotient, 3),
[ts_aux_sym_token2] = SHIFT(6),
[ts_aux_sym_token3] = SHIFT(8),
[ts_aux_sym_token4] = SHIFT(10),
},
[10] = {
[ts_aux_sym_token1] = REDUCE(ts_sym_quotient, 3),
[ts_aux_sym_token2] = REDUCE(ts_sym_quotient, 3),
[ts_aux_sym_token3] = SHIFT(7),
[ts_aux_sym_token4] = SHIFT(9),
[ts_aux_sym_token5] = SHIFT(11),
[ts_builtin_sym_end] = REDUCE(ts_sym_quotient, 3),
[ts_sym_expression] = SHIFT(11),
[ts_sym_sum] = SHIFT(12),
[ts_sym_difference] = SHIFT(12),
[ts_sym_product] = SHIFT(12),
[ts_sym_quotient] = SHIFT(12),
[ts_sym_exponent] = SHIFT(12),
[ts_sym_group] = SHIFT(12),
[ts_sym_number] = SHIFT(12),
[ts_sym_variable] = SHIFT(12),
[ts_aux_sym_token5] = SHIFT(13),
},
[11] = {
[ts_sym_difference] = SHIFT(1),
[ts_sym_exponent] = SHIFT(1),
[ts_sym_expression] = SHIFT(12),
[ts_sym_group] = SHIFT(1),
[ts_sym_number] = SHIFT(1),
[ts_sym_product] = SHIFT(1),
[ts_sym_quotient] = SHIFT(1),
[ts_sym_sum] = SHIFT(1),
[ts_sym_variable] = SHIFT(1),
[ts_aux_sym_token6] = SHIFT(13),
},
[12] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token0] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token1] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token2] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token3] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token4] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token5] = SHIFT(11),
[ts_builtin_sym_end] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token4] = SHIFT(10),
},
[13] = {
[ts_sym_difference] = SHIFT(14),
[ts_sym_exponent] = SHIFT(14),
[ts_sym_expression] = SHIFT(15),
[ts_sym_group] = SHIFT(14),
[ts_sym_number] = SHIFT(14),
[ts_sym_product] = SHIFT(14),
[ts_sym_quotient] = SHIFT(14),
[ts_sym_sum] = SHIFT(14),
[ts_sym_variable] = SHIFT(14),
[ts_aux_sym_token6] = SHIFT(26),
[ts_builtin_sym_error] = SHIFT(31),
},
[14] = {
[12] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token0] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token1] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token2] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token3] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token4] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token5] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token7] = REDUCE(ts_sym_expression, 1),
},
[13] = {
[ts_sym_expression] = SHIFT(14),
[ts_sym_sum] = SHIFT(25),
[ts_sym_difference] = SHIFT(25),
[ts_sym_product] = SHIFT(25),
[ts_sym_quotient] = SHIFT(25),
[ts_sym_exponent] = SHIFT(25),
[ts_sym_group] = SHIFT(25),
[ts_builtin_sym_error] = SHIFT(31),
[ts_sym_number] = SHIFT(25),
[ts_sym_variable] = SHIFT(25),
[ts_aux_sym_token5] = SHIFT(26),
},
[14] = {
[ts_aux_sym_token0] = SHIFT(15),
[ts_aux_sym_token1] = SHIFT(17),
[ts_aux_sym_token2] = SHIFT(19),
[ts_aux_sym_token3] = SHIFT(21),
[ts_aux_sym_token4] = SHIFT(23),
[ts_aux_sym_token6] = SHIFT(30),
},
[15] = {
[ts_aux_sym_token1] = SHIFT(16),
[ts_aux_sym_token2] = SHIFT(18),
[ts_aux_sym_token3] = SHIFT(20),
[ts_aux_sym_token4] = SHIFT(22),
[ts_aux_sym_token5] = SHIFT(24),
[ts_aux_sym_token7] = SHIFT(30),
[ts_sym_expression] = SHIFT(16),
[ts_sym_sum] = SHIFT(25),
[ts_sym_difference] = SHIFT(25),
[ts_sym_product] = SHIFT(25),
[ts_sym_quotient] = SHIFT(25),
[ts_sym_exponent] = SHIFT(25),
[ts_sym_group] = SHIFT(25),
[ts_sym_number] = SHIFT(25),
[ts_sym_variable] = SHIFT(25),
[ts_aux_sym_token5] = SHIFT(26),
},
[16] = {
[ts_sym_difference] = SHIFT(14),
[ts_sym_exponent] = SHIFT(14),
[ts_sym_expression] = SHIFT(17),
[ts_sym_group] = SHIFT(14),
[ts_sym_number] = SHIFT(14),
[ts_sym_product] = SHIFT(14),
[ts_sym_quotient] = SHIFT(14),
[ts_sym_sum] = SHIFT(14),
[ts_sym_variable] = SHIFT(14),
[ts_aux_sym_token6] = SHIFT(26),
[ts_aux_sym_token0] = SHIFT(15),
[ts_aux_sym_token1] = SHIFT(17),
[ts_aux_sym_token2] = SHIFT(19),
[ts_aux_sym_token3] = SHIFT(21),
[ts_aux_sym_token4] = SHIFT(23),
[ts_aux_sym_token6] = REDUCE(ts_sym_sum, 3),
},
[17] = {
[ts_aux_sym_token1] = SHIFT(16),
[ts_aux_sym_token2] = SHIFT(18),
[ts_aux_sym_token3] = SHIFT(20),
[ts_aux_sym_token4] = SHIFT(22),
[ts_aux_sym_token5] = SHIFT(24),
[ts_aux_sym_token7] = REDUCE(ts_sym_sum, 3),
[ts_sym_expression] = SHIFT(18),
[ts_sym_sum] = SHIFT(25),
[ts_sym_difference] = SHIFT(25),
[ts_sym_product] = SHIFT(25),
[ts_sym_quotient] = SHIFT(25),
[ts_sym_exponent] = SHIFT(25),
[ts_sym_group] = SHIFT(25),
[ts_sym_number] = SHIFT(25),
[ts_sym_variable] = SHIFT(25),
[ts_aux_sym_token5] = SHIFT(26),
},
[18] = {
[ts_sym_difference] = SHIFT(14),
[ts_sym_exponent] = SHIFT(14),
[ts_sym_expression] = SHIFT(19),
[ts_sym_group] = SHIFT(14),
[ts_sym_number] = SHIFT(14),
[ts_sym_product] = SHIFT(14),
[ts_sym_quotient] = SHIFT(14),
[ts_sym_sum] = SHIFT(14),
[ts_sym_variable] = SHIFT(14),
[ts_aux_sym_token6] = SHIFT(26),
[ts_aux_sym_token0] = SHIFT(15),
[ts_aux_sym_token1] = SHIFT(17),
[ts_aux_sym_token2] = SHIFT(19),
[ts_aux_sym_token3] = SHIFT(21),
[ts_aux_sym_token4] = SHIFT(23),
[ts_aux_sym_token6] = REDUCE(ts_sym_difference, 3),
},
[19] = {
[ts_aux_sym_token1] = SHIFT(16),
[ts_aux_sym_token2] = SHIFT(18),
[ts_aux_sym_token3] = SHIFT(20),
[ts_aux_sym_token4] = SHIFT(22),
[ts_aux_sym_token5] = SHIFT(24),
[ts_aux_sym_token7] = REDUCE(ts_sym_difference, 3),
[ts_sym_expression] = SHIFT(20),
[ts_sym_sum] = SHIFT(25),
[ts_sym_difference] = SHIFT(25),
[ts_sym_product] = SHIFT(25),
[ts_sym_quotient] = SHIFT(25),
[ts_sym_exponent] = SHIFT(25),
[ts_sym_group] = SHIFT(25),
[ts_sym_number] = SHIFT(25),
[ts_sym_variable] = SHIFT(25),
[ts_aux_sym_token5] = SHIFT(26),
},
[20] = {
[ts_sym_difference] = SHIFT(14),
[ts_sym_exponent] = SHIFT(14),
[ts_sym_expression] = SHIFT(21),
[ts_sym_group] = SHIFT(14),
[ts_sym_number] = SHIFT(14),
[ts_sym_product] = SHIFT(14),
[ts_sym_quotient] = SHIFT(14),
[ts_sym_sum] = SHIFT(14),
[ts_sym_variable] = SHIFT(14),
[ts_aux_sym_token6] = SHIFT(26),
[ts_aux_sym_token0] = REDUCE(ts_sym_product, 3),
[ts_aux_sym_token1] = REDUCE(ts_sym_product, 3),
[ts_aux_sym_token2] = SHIFT(19),
[ts_aux_sym_token3] = SHIFT(21),
[ts_aux_sym_token4] = SHIFT(23),
[ts_aux_sym_token6] = REDUCE(ts_sym_product, 3),
},
[21] = {
[ts_aux_sym_token1] = REDUCE(ts_sym_product, 3),
[ts_aux_sym_token2] = REDUCE(ts_sym_product, 3),
[ts_aux_sym_token3] = SHIFT(20),
[ts_aux_sym_token4] = SHIFT(22),
[ts_aux_sym_token5] = SHIFT(24),
[ts_aux_sym_token7] = REDUCE(ts_sym_product, 3),
[ts_sym_expression] = SHIFT(22),
[ts_sym_sum] = SHIFT(25),
[ts_sym_difference] = SHIFT(25),
[ts_sym_product] = SHIFT(25),
[ts_sym_quotient] = SHIFT(25),
[ts_sym_exponent] = SHIFT(25),
[ts_sym_group] = SHIFT(25),
[ts_sym_number] = SHIFT(25),
[ts_sym_variable] = SHIFT(25),
[ts_aux_sym_token5] = SHIFT(26),
},
[22] = {
[ts_sym_difference] = SHIFT(14),
[ts_sym_exponent] = SHIFT(14),
[ts_sym_expression] = SHIFT(23),
[ts_sym_group] = SHIFT(14),
[ts_sym_number] = SHIFT(14),
[ts_sym_product] = SHIFT(14),
[ts_sym_quotient] = SHIFT(14),
[ts_sym_sum] = SHIFT(14),
[ts_sym_variable] = SHIFT(14),
[ts_aux_sym_token6] = SHIFT(26),
[ts_aux_sym_token0] = REDUCE(ts_sym_quotient, 3),
[ts_aux_sym_token1] = REDUCE(ts_sym_quotient, 3),
[ts_aux_sym_token2] = SHIFT(19),
[ts_aux_sym_token3] = SHIFT(21),
[ts_aux_sym_token4] = SHIFT(23),
[ts_aux_sym_token6] = REDUCE(ts_sym_quotient, 3),
},
[23] = {
[ts_aux_sym_token1] = REDUCE(ts_sym_quotient, 3),
[ts_aux_sym_token2] = REDUCE(ts_sym_quotient, 3),
[ts_aux_sym_token3] = SHIFT(20),
[ts_aux_sym_token4] = SHIFT(22),
[ts_aux_sym_token5] = SHIFT(24),
[ts_aux_sym_token7] = REDUCE(ts_sym_quotient, 3),
[ts_sym_expression] = SHIFT(24),
[ts_sym_sum] = SHIFT(25),
[ts_sym_difference] = SHIFT(25),
[ts_sym_product] = SHIFT(25),
[ts_sym_quotient] = SHIFT(25),
[ts_sym_exponent] = SHIFT(25),
[ts_sym_group] = SHIFT(25),
[ts_sym_number] = SHIFT(25),
[ts_sym_variable] = SHIFT(25),
[ts_aux_sym_token5] = SHIFT(26),
},
[24] = {
[ts_sym_difference] = SHIFT(14),
[ts_sym_exponent] = SHIFT(14),
[ts_sym_expression] = SHIFT(25),
[ts_sym_group] = SHIFT(14),
[ts_sym_number] = SHIFT(14),
[ts_sym_product] = SHIFT(14),
[ts_sym_quotient] = SHIFT(14),
[ts_sym_sum] = SHIFT(14),
[ts_sym_variable] = SHIFT(14),
[ts_aux_sym_token6] = SHIFT(26),
},
[25] = {
[ts_aux_sym_token0] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token1] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token2] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token3] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token4] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token5] = SHIFT(24),
[ts_aux_sym_token7] = REDUCE(ts_sym_exponent, 3),
[ts_aux_sym_token4] = SHIFT(23),
[ts_aux_sym_token6] = REDUCE(ts_sym_exponent, 3),
},
[25] = {
[ts_aux_sym_token0] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token1] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token2] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token3] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token4] = REDUCE(ts_sym_expression, 1),
[ts_aux_sym_token6] = REDUCE(ts_sym_expression, 1),
},
[26] = {
[ts_sym_difference] = SHIFT(14),
[ts_sym_exponent] = SHIFT(14),
[ts_sym_expression] = SHIFT(27),
[ts_sym_group] = SHIFT(14),
[ts_sym_number] = SHIFT(14),
[ts_sym_product] = SHIFT(14),
[ts_sym_quotient] = SHIFT(14),
[ts_sym_sum] = SHIFT(14),
[ts_sym_variable] = SHIFT(14),
[ts_aux_sym_token6] = SHIFT(26),
[ts_sym_sum] = SHIFT(25),
[ts_sym_difference] = SHIFT(25),
[ts_sym_product] = SHIFT(25),
[ts_sym_quotient] = SHIFT(25),
[ts_sym_exponent] = SHIFT(25),
[ts_sym_group] = SHIFT(25),
[ts_builtin_sym_error] = SHIFT(29),
[ts_sym_number] = SHIFT(25),
[ts_sym_variable] = SHIFT(25),
[ts_aux_sym_token5] = SHIFT(26),
},
[27] = {
[ts_aux_sym_token1] = SHIFT(16),
[ts_aux_sym_token2] = SHIFT(18),
[ts_aux_sym_token3] = SHIFT(20),
[ts_aux_sym_token4] = SHIFT(22),
[ts_aux_sym_token5] = SHIFT(24),
[ts_aux_sym_token7] = SHIFT(28),
[ts_aux_sym_token0] = SHIFT(15),
[ts_aux_sym_token1] = SHIFT(17),
[ts_aux_sym_token2] = SHIFT(19),
[ts_aux_sym_token3] = SHIFT(21),
[ts_aux_sym_token4] = SHIFT(23),
[ts_aux_sym_token6] = SHIFT(28),
},
[28] = {
[ts_aux_sym_token0] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token1] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token2] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token3] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token4] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token5] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token7] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token6] = REDUCE(ts_sym_group, 3),
},
[29] = {
[ts_aux_sym_token7] = SHIFT(28),
[ts_aux_sym_token6] = SHIFT(28),
},
[30] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token0] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token1] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token2] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token3] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token4] = REDUCE(ts_sym_group, 3),
[ts_aux_sym_token5] = REDUCE(ts_sym_group, 3),
[ts_builtin_sym_end] = REDUCE(ts_sym_group, 3),
},
[31] = {
[ts_aux_sym_token7] = SHIFT(30),
[ts_aux_sym_token6] = SHIFT(30),
},
};

3801
examples/parsers/golang.c
View file

File diff suppressed because it is too large Load diff

14629
examples/parsers/javascript.c
View file

File diff suppressed because it is too large Load diff

793
examples/parsers/json.c
View file

@ -4,54 +4,54 @@
#define SYMBOL_COUNT 18
enum {
ts_sym_array = 2,
ts_sym_false = 3,
ts_sym_null = 4,
ts_sym_number = 5,
ts_sym_object = 6,
ts_sym_string = 7,
ts_sym_value = 2,
ts_sym_object = 3,
ts_sym_array = 4,
ts_sym_string = 5,
ts_sym_number = 6,
ts_sym_null = 7,
ts_sym_true = 8,
ts_sym_value = 9,
ts_aux_sym_array_repeat1 = 10,
ts_aux_sym_object_repeat1 = 11,
ts_aux_sym_token1 = 12,
ts_aux_sym_token2 = 13,
ts_aux_sym_token3 = 14,
ts_aux_sym_token4 = 15,
ts_aux_sym_token5 = 16,
ts_aux_sym_token6 = 17,
ts_sym_false = 9,
ts_aux_sym_object_repeat0 = 10,
ts_aux_sym_array_repeat0 = 11,
ts_aux_sym_token0 = 12,
ts_aux_sym_token1 = 13,
ts_aux_sym_token2 = 14,
ts_aux_sym_token3 = 15,
ts_aux_sym_token4 = 16,
ts_aux_sym_token5 = 17,
};
SYMBOL_NAMES = {
[ts_sym_array] = "array",
[ts_sym_false] = "false",
[ts_sym_null] = "null",
[ts_sym_number] = "number",
[ts_sym_object] = "object",
[ts_sym_string] = "string",
[ts_sym_true] = "true",
[ts_sym_value] = "value",
[ts_aux_sym_array_repeat1] = "array_repeat1",
[ts_aux_sym_object_repeat1] = "object_repeat1",
[ts_aux_sym_token1] = "'{'",
[ts_aux_sym_token2] = "':'",
[ts_aux_sym_token3] = "','",
[ts_aux_sym_token4] = "'}'",
[ts_aux_sym_token5] = "'['",
[ts_aux_sym_token6] = "']'",
[ts_builtin_sym_end] = "EOF",
[ts_builtin_sym_error] = "ERROR",
[ts_sym_object] = "object",
[ts_sym_array] = "array",
[ts_builtin_sym_error] = "error",
[ts_builtin_sym_end] = "end",
[ts_sym_string] = "string",
[ts_sym_number] = "number",
[ts_sym_null] = "null",
[ts_sym_true] = "true",
[ts_sym_false] = "false",
[ts_aux_sym_object_repeat0] = "object_repeat0",
[ts_aux_sym_array_repeat0] = "array_repeat0",
[ts_aux_sym_token0] = "'{'",
[ts_aux_sym_token1] = "':'",
[ts_aux_sym_token2] = "','",
[ts_aux_sym_token3] = "'}'",
[ts_aux_sym_token4] = "'['",
[ts_aux_sym_token5] = "']'",
};
HIDDEN_SYMBOLS = {
[ts_aux_sym_array_repeat1] = 1,
[ts_aux_sym_object_repeat1] = 1,
[ts_aux_sym_object_repeat0] = 1,
[ts_aux_sym_array_repeat0] = 1,
[ts_aux_sym_token0] = 1,
[ts_aux_sym_token1] = 1,
[ts_aux_sym_token2] = 1,
[ts_aux_sym_token3] = 1,
[ts_aux_sym_token4] = 1,
[ts_aux_sym_token5] = 1,
[ts_aux_sym_token6] = 1,
};
LEX_FN() {
@ -71,49 +71,49 @@ LEX_FN() {
ACCEPT_TOKEN(ts_builtin_sym_end);
case 2:
START_TOKEN();
if (('\t' <= lookahead && lookahead <= '\n') ||
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(2);
if (lookahead == ',')
ADVANCE(3);
if (lookahead == '}')
ADVANCE(4);
ADVANCE(3);
LEX_ERROR();
case 3:
ACCEPT_TOKEN(ts_aux_sym_token3);
case 4:
ACCEPT_TOKEN(ts_aux_sym_token4);
case 5:
START_TOKEN();
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(5);
if (lookahead == '}')
ADVANCE(4);
LEX_ERROR();
case 6:
START_TOKEN();
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(6);
ADVANCE(4);
if (lookahead == ',')
ADVANCE(5);
if (lookahead == '}')
ADVANCE(3);
if (lookahead == ']')
ADVANCE(7);
LEX_ERROR();
case 7:
ACCEPT_TOKEN(ts_aux_sym_token6);
case 8:
case 5:
ACCEPT_TOKEN(ts_aux_sym_token2);
case 6:
START_TOKEN();
if ((lookahead == '\t') ||
(lookahead == '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(6);
if (lookahead == ']')
ADVANCE(7);
LEX_ERROR();
case 7:
ACCEPT_TOKEN(ts_aux_sym_token5);
case 8:
START_TOKEN();
if (('\t' <= lookahead && lookahead <= '\n') ||
(lookahead == '\r') ||
(lookahead == ' '))
ADVANCE(8);
if (lookahead == ',')
ADVANCE(5);
if (lookahead == ']')
ADVANCE(7);
LEX_ERROR();
@ -189,7 +189,7 @@ LEX_FN() {
ADVANCE(17);
ACCEPT_TOKEN(ts_sym_number);
case 18:
ACCEPT_TOKEN(ts_aux_sym_token5);
ACCEPT_TOKEN(ts_aux_sym_token4);
case 19:
if (lookahead == 'a')
ADVANCE(20);
@ -237,7 +237,7 @@ LEX_FN() {
case 31:
ACCEPT_TOKEN(ts_sym_true);
case 32:
ACCEPT_TOKEN(ts_aux_sym_token1);
ACCEPT_TOKEN(ts_aux_sym_token0);
case 33:
START_TOKEN();
if ((lookahead == '\t') ||
@ -249,7 +249,7 @@ LEX_FN() {
ADVANCE(34);
LEX_ERROR();
case 34:
ACCEPT_TOKEN(ts_aux_sym_token2);
ACCEPT_TOKEN(ts_aux_sym_token1);
case 35:
START_TOKEN();
if (('\t' <= lookahead && lookahead <= '\n') ||
@ -259,7 +259,7 @@ LEX_FN() {
if (lookahead == '\"')
ADVANCE(10);
if (lookahead == '}')
ADVANCE(4);
ADVANCE(3);
LEX_ERROR();
case 36:
START_TOKEN();
@ -305,7 +305,7 @@ LEX_FN() {
if (lookahead == '\"')
ADVANCE(10);
if (lookahead == ',')
ADVANCE(3);
ADVANCE(5);
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(15);
if (lookahead == ':')
@ -323,10 +323,9 @@ LEX_FN() {
if (lookahead == '{')
ADVANCE(32);
if (lookahead == '}')
ADVANCE(4);
ADVANCE(3);
LEX_ERROR();
case ts_lex_state_error:
START_TOKEN();
if (lookahead == '\0')
ADVANCE(1);
if (('\t' <= lookahead && lookahead <= '\n') ||
@ -336,7 +335,7 @@ LEX_FN() {
if (lookahead == '\"')
ADVANCE(10);
if (lookahead == ',')
ADVANCE(3);
ADVANCE(5);
if ('0' <= lookahead && lookahead <= '9')
ADVANCE(15);
if (lookahead == ':')
@ -354,7 +353,7 @@ LEX_FN() {
if (lookahead == '{')
ADVANCE(32);
if (lookahead == '}')
ADVANCE(4);
ADVANCE(3);
LEX_ERROR();
default:
LEX_PANIC();
@ -366,60 +365,60 @@ LEX_STATES = {
[1] = 0,
[2] = 0,
[3] = 35,
[4] = 33,
[5] = 9,
[6] = 2,
[7] = 2,
[8] = 5,
[9] = 0,
[10] = 37,
[11] = 33,
[12] = 9,
[4] = 4,
[5] = 2,
[6] = 0,
[7] = 37,
[8] = 4,
[9] = 2,
[10] = 33,
[11] = 9,
[12] = 4,
[13] = 2,
[14] = 5,
[14] = 4,
[15] = 35,
[16] = 33,
[17] = 9,
[18] = 2,
[19] = 5,
[20] = 2,
[21] = 36,
[22] = 6,
[23] = 6,
[24] = 8,
[25] = 2,
[26] = 9,
[27] = 6,
[28] = 8,
[29] = 35,
[30] = 33,
[31] = 9,
[32] = 2,
[33] = 5,
[34] = 6,
[35] = 6,
[36] = 2,
[37] = 5,
[38] = 6,
[39] = 36,
[40] = 6,
[16] = 4,
[17] = 2,
[18] = 4,
[19] = 33,
[20] = 9,
[21] = 4,
[22] = 2,
[23] = 4,
[24] = 36,
[25] = 8,
[26] = 6,
[27] = 4,
[28] = 9,
[29] = 8,
[30] = 6,
[31] = 8,
[32] = 35,
[33] = 4,
[34] = 2,
[35] = 8,
[36] = 33,
[37] = 9,
[38] = 4,
[39] = 2,
[40] = 8,
[41] = 8,
[42] = 6,
[43] = 6,
[44] = 2,
[45] = 2,
[46] = 2,
[47] = 5,
[48] = 2,
[49] = 2,
[50] = 5,
[51] = 0,
[42] = 36,
[43] = 8,
[44] = 6,
[45] = 8,
[46] = 8,
[47] = 4,
[48] = 4,
[49] = 33,
[50] = 9,
[51] = 4,
[52] = 2,
[53] = 5,
[53] = 0,
[54] = 0,
[55] = 36,
[56] = 6,
[57] = 8,
[56] = 8,
[57] = 6,
[58] = 0,
[59] = 0,
};
@ -429,309 +428,309 @@ LEX_STATES = {
PARSE_TABLE = {
[0] = {
[ts_sym_array] = SHIFT(1),
[ts_sym_false] = SHIFT(1),
[ts_sym_null] = SHIFT(1),
[ts_sym_number] = SHIFT(1),
[ts_sym_object] = SHIFT(1),
[ts_sym_string] = SHIFT(1),
[ts_sym_true] = SHIFT(1),
[ts_sym_value] = SHIFT(2),
[ts_aux_sym_token1] = SHIFT(3),
[ts_aux_sym_token5] = SHIFT(55),
[ts_sym_value] = SHIFT(1),
[ts_sym_object] = SHIFT(2),
[ts_sym_array] = SHIFT(2),
[ts_sym_string] = SHIFT(2),
[ts_sym_number] = SHIFT(2),
[ts_sym_null] = SHIFT(2),
[ts_sym_true] = SHIFT(2),
[ts_sym_false] = SHIFT(2),
[ts_aux_sym_token0] = SHIFT(3),
[ts_aux_sym_token4] = SHIFT(55),
},
[1] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_value, 1),
},
[2] = {
[ts_builtin_sym_end] = ACCEPT_INPUT(),
},
[2] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_value, 1),
},
[3] = {
[ts_sym_string] = SHIFT(4),
[ts_aux_sym_token4] = SHIFT(51),
[ts_builtin_sym_error] = SHIFT(52),
[ts_builtin_sym_error] = SHIFT(4),
[ts_sym_string] = SHIFT(49),
[ts_aux_sym_token3] = SHIFT(54),
},
[4] = {
[ts_aux_sym_token2] = SHIFT(5),
[ts_aux_sym_object_repeat0] = SHIFT(5),
[ts_aux_sym_token2] = SHIFT(7),
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 0),
},
[5] = {
[ts_sym_array] = SHIFT(6),
[ts_sym_false] = SHIFT(6),
[ts_sym_null] = SHIFT(6),
[ts_sym_number] = SHIFT(6),
[ts_sym_object] = SHIFT(6),
[ts_sym_string] = SHIFT(6),
[ts_sym_true] = SHIFT(6),
[ts_sym_value] = SHIFT(7),
[ts_aux_sym_token1] = SHIFT(15),
[ts_aux_sym_token5] = SHIFT(21),
[ts_aux_sym_token3] = SHIFT(6),
},
[6] = {
[ts_aux_sym_token3] = REDUCE(ts_sym_value, 1),
[ts_aux_sym_token4] = REDUCE(ts_sym_value, 1),
},
[7] = {
[ts_aux_sym_object_repeat1] = SHIFT(8),
[ts_aux_sym_token3] = SHIFT(10),
[ts_aux_sym_token4] = REDUCE(ts_aux_sym_object_repeat1, 0),
},
[8] = {
[ts_aux_sym_token4] = SHIFT(9),
},
[9] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_object, 6),
},
[10] = {
[ts_sym_string] = SHIFT(11),
[ts_builtin_sym_error] = SHIFT(49),
},
[11] = {
[ts_aux_sym_token2] = SHIFT(12),
},
[12] = {
[ts_sym_array] = SHIFT(6),
[ts_sym_false] = SHIFT(6),
[ts_sym_null] = SHIFT(6),
[ts_sym_number] = SHIFT(6),
[ts_sym_object] = SHIFT(6),
[ts_sym_string] = SHIFT(6),
[ts_sym_true] = SHIFT(6),
[ts_sym_value] = SHIFT(13),
[ts_aux_sym_token1] = SHIFT(15),
[ts_aux_sym_token5] = SHIFT(21),
},
[13] = {
[ts_aux_sym_object_repeat1] = SHIFT(14),
[ts_aux_sym_token3] = SHIFT(10),
[ts_aux_sym_token4] = REDUCE(ts_aux_sym_object_repeat1, 0),
},
[14] = {
[ts_aux_sym_token4] = REDUCE(ts_aux_sym_object_repeat1, 5),
},
[15] = {
[ts_sym_string] = SHIFT(16),
[ts_aux_sym_token4] = SHIFT(45),
[ts_builtin_sym_error] = SHIFT(46),
},
[16] = {
[ts_aux_sym_token2] = SHIFT(17),
},
[17] = {
[ts_sym_array] = SHIFT(6),
[ts_sym_false] = SHIFT(6),
[ts_sym_null] = SHIFT(6),
[ts_sym_number] = SHIFT(6),
[ts_sym_object] = SHIFT(6),
[ts_sym_string] = SHIFT(6),
[ts_sym_true] = SHIFT(6),
[ts_sym_value] = SHIFT(18),
[ts_aux_sym_token1] = SHIFT(15),
[ts_aux_sym_token5] = SHIFT(21),
},
[18] = {
[ts_aux_sym_object_repeat1] = SHIFT(19),
[ts_aux_sym_token3] = SHIFT(10),
[ts_aux_sym_token4] = REDUCE(ts_aux_sym_object_repeat1, 0),
},
[19] = {
[ts_aux_sym_token4] = SHIFT(20),
},
[20] = {
[ts_aux_sym_token3] = REDUCE(ts_sym_object, 6),
[ts_aux_sym_token4] = REDUCE(ts_sym_object, 6),
},
[21] = {
[ts_sym_array] = SHIFT(22),
[ts_sym_false] = SHIFT(22),
[ts_sym_null] = SHIFT(22),
[ts_sym_number] = SHIFT(22),
[ts_sym_object] = SHIFT(22),
[ts_sym_string] = SHIFT(22),
[ts_sym_true] = SHIFT(22),
[ts_sym_value] = SHIFT(23),
[ts_aux_sym_token1] = SHIFT(29),
[ts_aux_sym_token5] = SHIFT(39),
[ts_aux_sym_token6] = SHIFT(44),
[ts_builtin_sym_error] = SHIFT(23),
},
[22] = {
[ts_aux_sym_token3] = REDUCE(ts_sym_value, 1),
[ts_aux_sym_token6] = REDUCE(ts_sym_value, 1),
},
[23] = {
[ts_aux_sym_array_repeat1] = SHIFT(24),
[ts_aux_sym_token3] = SHIFT(26),
[ts_aux_sym_token6] = REDUCE(ts_aux_sym_array_repeat1, 0),
},
[24] = {
[ts_aux_sym_token6] = SHIFT(25),
},
[25] = {
[ts_aux_sym_token3] = REDUCE(ts_sym_array, 4),
[ts_aux_sym_token4] = REDUCE(ts_sym_array, 4),
},
[26] = {
[ts_sym_array] = SHIFT(22),
[ts_sym_false] = SHIFT(22),
[ts_sym_null] = SHIFT(22),
[ts_sym_number] = SHIFT(22),
[ts_sym_object] = SHIFT(22),
[ts_sym_string] = SHIFT(22),
[ts_sym_true] = SHIFT(22),
[ts_sym_value] = SHIFT(27),
[ts_aux_sym_token1] = SHIFT(29),
[ts_aux_sym_token5] = SHIFT(39),
[ts_builtin_sym_error] = SHIFT(27),
},
[27] = {
[ts_aux_sym_array_repeat1] = SHIFT(28),
[ts_aux_sym_token3] = SHIFT(26),
[ts_aux_sym_token6] = REDUCE(ts_aux_sym_array_repeat1, 0),
},
[28] = {
[ts_aux_sym_token6] = REDUCE(ts_aux_sym_array_repeat1, 3),
},
[29] = {
[ts_sym_string] = SHIFT(30),
[ts_aux_sym_token4] = SHIFT(35),
[ts_builtin_sym_error] = SHIFT(36),
},
[30] = {
[ts_aux_sym_token2] = SHIFT(31),
},
[31] = {
[ts_sym_array] = SHIFT(6),
[ts_sym_false] = SHIFT(6),
[ts_sym_null] = SHIFT(6),
[ts_sym_number] = SHIFT(6),
[ts_sym_object] = SHIFT(6),
[ts_sym_string] = SHIFT(6),
[ts_sym_true] = SHIFT(6),
[ts_sym_value] = SHIFT(32),
[ts_aux_sym_token1] = SHIFT(15),
[ts_aux_sym_token5] = SHIFT(21),
},
[32] = {
[ts_aux_sym_object_repeat1] = SHIFT(33),
[ts_aux_sym_token3] = SHIFT(10),
[ts_aux_sym_token4] = REDUCE(ts_aux_sym_object_repeat1, 0),
},
[33] = {
[ts_aux_sym_token4] = SHIFT(34),
},
[34] = {
[ts_aux_sym_token3] = REDUCE(ts_sym_object, 6),
[ts_aux_sym_token6] = REDUCE(ts_sym_object, 6),
},
[35] = {
[ts_aux_sym_token3] = REDUCE(ts_sym_object, 2),
[ts_aux_sym_token6] = REDUCE(ts_sym_object, 2),
},
[36] = {
[ts_aux_sym_object_repeat1] = SHIFT(37),
[ts_aux_sym_token3] = SHIFT(10),
[ts_aux_sym_token4] = REDUCE(ts_aux_sym_object_repeat1, 0),
},
[37] = {
[ts_aux_sym_token4] = SHIFT(38),
},
[38] = {
[ts_aux_sym_token3] = REDUCE(ts_sym_object, 4),
[ts_aux_sym_token6] = REDUCE(ts_sym_object, 4),
},
[39] = {
[ts_sym_array] = SHIFT(22),
[ts_sym_false] = SHIFT(22),
[ts_sym_null] = SHIFT(22),
[ts_sym_number] = SHIFT(22),
[ts_sym_object] = SHIFT(22),
[ts_sym_string] = SHIFT(22),
[ts_sym_true] = SHIFT(22),
[ts_sym_value] = SHIFT(40),
[ts_aux_sym_token1] = SHIFT(29),
[ts_aux_sym_token5] = SHIFT(39),
[ts_aux_sym_token6] = SHIFT(43),
[ts_builtin_sym_error] = SHIFT(40),
},
[40] = {
[ts_aux_sym_array_repeat1] = SHIFT(41),
[ts_aux_sym_token3] = SHIFT(26),
[ts_aux_sym_token6] = REDUCE(ts_aux_sym_array_repeat1, 0),
},
[41] = {
[ts_aux_sym_token6] = SHIFT(42),
},
[42] = {
[ts_aux_sym_token3] = REDUCE(ts_sym_array, 4),
[ts_aux_sym_token6] = REDUCE(ts_sym_array, 4),
},
[43] = {
[ts_aux_sym_token3] = REDUCE(ts_sym_array, 2),
[ts_aux_sym_token6] = REDUCE(ts_sym_array, 2),
},
[44] = {
[ts_aux_sym_token3] = REDUCE(ts_sym_array, 2),
[ts_aux_sym_token4] = REDUCE(ts_sym_array, 2),
},
[45] = {
[ts_aux_sym_token3] = REDUCE(ts_sym_object, 2),
[ts_aux_sym_token4] = REDUCE(ts_sym_object, 2),
},
[46] = {
[ts_aux_sym_object_repeat1] = SHIFT(47),
[ts_aux_sym_token3] = SHIFT(10),
[ts_aux_sym_token4] = REDUCE(ts_aux_sym_object_repeat1, 0),
},
[47] = {
[ts_aux_sym_token4] = SHIFT(48),
},
[48] = {
[ts_aux_sym_token3] = REDUCE(ts_sym_object, 4),
[ts_aux_sym_token4] = REDUCE(ts_sym_object, 4),
},
[49] = {
[ts_aux_sym_object_repeat1] = SHIFT(50),
[ts_aux_sym_token3] = SHIFT(10),
[ts_aux_sym_token4] = REDUCE(ts_aux_sym_object_repeat1, 0),
},
[50] = {
[ts_aux_sym_token4] = REDUCE(ts_aux_sym_object_repeat1, 3),
},
[51] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_object, 2),
},
[52] = {
[ts_aux_sym_object_repeat1] = SHIFT(53),
[ts_aux_sym_token3] = SHIFT(10),
[ts_aux_sym_token4] = REDUCE(ts_aux_sym_object_repeat1, 0),
},
[53] = {
[ts_aux_sym_token4] = SHIFT(54),
},
[54] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_object, 4),
},
[7] = {
[ts_builtin_sym_error] = SHIFT(8),
[ts_sym_string] = SHIFT(10),
},
[8] = {
[ts_aux_sym_object_repeat0] = SHIFT(9),
[ts_aux_sym_token2] = SHIFT(7),
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 0),
},
[9] = {
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 3),
},
[10] = {
[ts_aux_sym_token1] = SHIFT(11),
},
[11] = {
[ts_sym_value] = SHIFT(12),
[ts_sym_object] = SHIFT(14),
[ts_sym_array] = SHIFT(14),
[ts_sym_string] = SHIFT(14),
[ts_sym_number] = SHIFT(14),
[ts_sym_null] = SHIFT(14),
[ts_sym_true] = SHIFT(14),
[ts_sym_false] = SHIFT(14),
[ts_aux_sym_token0] = SHIFT(15),
[ts_aux_sym_token4] = SHIFT(24),
},
[12] = {
[ts_aux_sym_object_repeat0] = SHIFT(13),
[ts_aux_sym_token2] = SHIFT(7),
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 0),
},
[13] = {
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 5),
},
[14] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_value, 1),
[ts_aux_sym_token3] = REDUCE(ts_sym_value, 1),
},
[15] = {
[ts_builtin_sym_error] = SHIFT(16),
[ts_sym_string] = SHIFT(19),
[ts_aux_sym_token3] = SHIFT(48),
},
[16] = {
[ts_aux_sym_object_repeat0] = SHIFT(17),
[ts_aux_sym_token2] = SHIFT(7),
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 0),
},
[17] = {
[ts_aux_sym_token3] = SHIFT(18),
},
[18] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_object, 4),
[ts_aux_sym_token3] = REDUCE(ts_sym_object, 4),
},
[19] = {
[ts_aux_sym_token1] = SHIFT(20),
},
[20] = {
[ts_sym_value] = SHIFT(21),
[ts_sym_object] = SHIFT(14),
[ts_sym_array] = SHIFT(14),
[ts_sym_string] = SHIFT(14),
[ts_sym_number] = SHIFT(14),
[ts_sym_null] = SHIFT(14),
[ts_sym_true] = SHIFT(14),
[ts_sym_false] = SHIFT(14),
[ts_aux_sym_token0] = SHIFT(15),
[ts_aux_sym_token4] = SHIFT(24),
},
[21] = {
[ts_aux_sym_object_repeat0] = SHIFT(22),
[ts_aux_sym_token2] = SHIFT(7),
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 0),
},
[22] = {
[ts_aux_sym_token3] = SHIFT(23),
},
[23] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_object, 6),
[ts_aux_sym_token3] = REDUCE(ts_sym_object, 6),
},
[24] = {
[ts_sym_value] = SHIFT(25),
[ts_sym_object] = SHIFT(31),
[ts_sym_array] = SHIFT(31),
[ts_builtin_sym_error] = SHIFT(25),
[ts_sym_string] = SHIFT(31),
[ts_sym_number] = SHIFT(31),
[ts_sym_null] = SHIFT(31),
[ts_sym_true] = SHIFT(31),
[ts_sym_false] = SHIFT(31),
[ts_aux_sym_token0] = SHIFT(32),
[ts_aux_sym_token4] = SHIFT(42),
[ts_aux_sym_token5] = SHIFT(47),
},
[25] = {
[ts_aux_sym_array_repeat0] = SHIFT(26),
[ts_aux_sym_token2] = SHIFT(28),
[ts_aux_sym_token5] = REDUCE(ts_aux_sym_array_repeat0, 0),
},
[26] = {
[ts_aux_sym_token5] = SHIFT(27),
},
[27] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_array, 4),
[ts_aux_sym_token3] = REDUCE(ts_sym_array, 4),
},
[28] = {
[ts_sym_value] = SHIFT(29),
[ts_sym_object] = SHIFT(31),
[ts_sym_array] = SHIFT(31),
[ts_builtin_sym_error] = SHIFT(29),
[ts_sym_string] = SHIFT(31),
[ts_sym_number] = SHIFT(31),
[ts_sym_null] = SHIFT(31),
[ts_sym_true] = SHIFT(31),
[ts_sym_false] = SHIFT(31),
[ts_aux_sym_token0] = SHIFT(32),
[ts_aux_sym_token4] = SHIFT(42),
},
[29] = {
[ts_aux_sym_array_repeat0] = SHIFT(30),
[ts_aux_sym_token2] = SHIFT(28),
[ts_aux_sym_token5] = REDUCE(ts_aux_sym_array_repeat0, 0),
},
[30] = {
[ts_aux_sym_token5] = REDUCE(ts_aux_sym_array_repeat0, 3),
},
[31] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_value, 1),
[ts_aux_sym_token5] = REDUCE(ts_sym_value, 1),
},
[32] = {
[ts_builtin_sym_error] = SHIFT(33),
[ts_sym_string] = SHIFT(36),
[ts_aux_sym_token3] = SHIFT(41),
},
[33] = {
[ts_aux_sym_object_repeat0] = SHIFT(34),
[ts_aux_sym_token2] = SHIFT(7),
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 0),
},
[34] = {
[ts_aux_sym_token3] = SHIFT(35),
},
[35] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_object, 4),
[ts_aux_sym_token5] = REDUCE(ts_sym_object, 4),
},
[36] = {
[ts_aux_sym_token1] = SHIFT(37),
},
[37] = {
[ts_sym_value] = SHIFT(38),
[ts_sym_object] = SHIFT(14),
[ts_sym_array] = SHIFT(14),
[ts_sym_string] = SHIFT(14),
[ts_sym_number] = SHIFT(14),
[ts_sym_null] = SHIFT(14),
[ts_sym_true] = SHIFT(14),
[ts_sym_false] = SHIFT(14),
[ts_aux_sym_token0] = SHIFT(15),
[ts_aux_sym_token4] = SHIFT(24),
},
[38] = {
[ts_aux_sym_object_repeat0] = SHIFT(39),
[ts_aux_sym_token2] = SHIFT(7),
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 0),
},
[39] = {
[ts_aux_sym_token3] = SHIFT(40),
},
[40] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_object, 6),
[ts_aux_sym_token5] = REDUCE(ts_sym_object, 6),
},
[41] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_object, 2),
[ts_aux_sym_token5] = REDUCE(ts_sym_object, 2),
},
[42] = {
[ts_sym_value] = SHIFT(43),
[ts_sym_object] = SHIFT(31),
[ts_sym_array] = SHIFT(31),
[ts_builtin_sym_error] = SHIFT(43),
[ts_sym_string] = SHIFT(31),
[ts_sym_number] = SHIFT(31),
[ts_sym_null] = SHIFT(31),
[ts_sym_true] = SHIFT(31),
[ts_sym_false] = SHIFT(31),
[ts_aux_sym_token0] = SHIFT(32),
[ts_aux_sym_token4] = SHIFT(42),
[ts_aux_sym_token5] = SHIFT(46),
},
[43] = {
[ts_aux_sym_array_repeat0] = SHIFT(44),
[ts_aux_sym_token2] = SHIFT(28),
[ts_aux_sym_token5] = REDUCE(ts_aux_sym_array_repeat0, 0),
},
[44] = {
[ts_aux_sym_token5] = SHIFT(45),
},
[45] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_array, 4),
[ts_aux_sym_token5] = REDUCE(ts_sym_array, 4),
},
[46] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_array, 2),
[ts_aux_sym_token5] = REDUCE(ts_sym_array, 2),
},
[47] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_array, 2),
[ts_aux_sym_token3] = REDUCE(ts_sym_array, 2),
},
[48] = {
[ts_aux_sym_token2] = REDUCE(ts_sym_object, 2),
[ts_aux_sym_token3] = REDUCE(ts_sym_object, 2),
},
[49] = {
[ts_aux_sym_token1] = SHIFT(50),
},
[50] = {
[ts_sym_value] = SHIFT(51),
[ts_sym_object] = SHIFT(14),
[ts_sym_array] = SHIFT(14),
[ts_sym_string] = SHIFT(14),
[ts_sym_number] = SHIFT(14),
[ts_sym_null] = SHIFT(14),
[ts_sym_true] = SHIFT(14),
[ts_sym_false] = SHIFT(14),
[ts_aux_sym_token0] = SHIFT(15),
[ts_aux_sym_token4] = SHIFT(24),
},
[51] = {
[ts_aux_sym_object_repeat0] = SHIFT(52),
[ts_aux_sym_token2] = SHIFT(7),
[ts_aux_sym_token3] = REDUCE(ts_aux_sym_object_repeat0, 0),
},
[52] = {
[ts_aux_sym_token3] = SHIFT(53),
},
[53] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_object, 6),
},
[54] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_object, 2),
},
[55] = {
[ts_sym_array] = SHIFT(22),
[ts_sym_false] = SHIFT(22),
[ts_sym_null] = SHIFT(22),
[ts_sym_number] = SHIFT(22),
[ts_sym_object] = SHIFT(22),
[ts_sym_string] = SHIFT(22),
[ts_sym_true] = SHIFT(22),
[ts_sym_value] = SHIFT(56),
[ts_aux_sym_token1] = SHIFT(29),
[ts_aux_sym_token5] = SHIFT(39),
[ts_aux_sym_token6] = SHIFT(59),
[ts_sym_object] = SHIFT(31),
[ts_sym_array] = SHIFT(31),
[ts_builtin_sym_error] = SHIFT(56),
[ts_sym_string] = SHIFT(31),
[ts_sym_number] = SHIFT(31),
[ts_sym_null] = SHIFT(31),
[ts_sym_true] = SHIFT(31),
[ts_sym_false] = SHIFT(31),
[ts_aux_sym_token0] = SHIFT(32),
[ts_aux_sym_token4] = SHIFT(42),
[ts_aux_sym_token5] = SHIFT(59),
},
[56] = {
[ts_aux_sym_array_repeat1] = SHIFT(57),
[ts_aux_sym_token3] = SHIFT(26),
[ts_aux_sym_token6] = REDUCE(ts_aux_sym_array_repeat1, 0),
[ts_aux_sym_array_repeat0] = SHIFT(57),
[ts_aux_sym_token2] = SHIFT(28),
[ts_aux_sym_token5] = REDUCE(ts_aux_sym_array_repeat0, 0),
},
[57] = {
[ts_aux_sym_token6] = SHIFT(58),
[ts_aux_sym_token5] = SHIFT(58),
},
[58] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_array, 4),

15
spec/compiler/build_tables/conflict_manager_spec.cc
View file

@ -21,12 +21,7 @@ describe("resolving parse conflicts", []() {
}, {});
before_each([&]() {
manager = new ConflictManager(parse_grammar, lex_grammar, {
{ Symbol("rule1"), "rule1" },
{ Symbol("rule2"), "rule2" },
{ Symbol("token1"), "token1" },
{ Symbol("token2"), "token2" },
});
manager = new ConflictManager(parse_grammar, lex_grammar);
});
after_each([&]() {
@ -34,8 +29,8 @@ describe("resolving parse conflicts", []() {
});
describe("lexical conflicts", [&]() {
Symbol sym1("token1");
Symbol sym2("token2");
ISymbol sym1(1, SymbolOptionToken);
ISymbol sym2(2, SymbolOptionToken);
it("favors non-errors over lexical errors", [&]() {
should_update = manager->resolve_lex_action(LexAction::Error(), LexAction::Advance(2));
@ -55,8 +50,8 @@ describe("resolving parse conflicts", []() {
});
describe("syntactic conflicts", [&]() {
Symbol sym1("rule1");
Symbol sym2("rule2");
ISymbol sym1(0);
ISymbol sym2(1);
it("favors non-errors over parse errors", [&]() {
should_update = manager->resolve_parse_action(sym1, ParseAction::Error(), ParseAction::Shift(2, { 0 }));

78
spec/compiler/build_tables/first_set_spec.cc
View file

@ -14,58 +14,60 @@ describe("computing FIRST sets", []() {
describe("for a sequence AB", [&]() {
it("ignores B when A cannot be blank", [&]() {
auto rule = seq({ sym("x"), sym("y") });
auto rule = seq({ i_token(0), i_token(1) });
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol("x"),
AssertThat(first_set(rule, null_grammar), Equals(set<ISymbol>({
ISymbol(0, SymbolOptionToken),
})));
});
it("includes FIRST(B) when A can be blank", [&]() {
auto rule = seq({
choice({
sym("x"),
i_token(0),
blank() }),
sym("y") });
i_token(1) });
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol("x"),
Symbol("y")
AssertThat(first_set(rule, null_grammar), Equals(set<ISymbol>({
ISymbol(0, SymbolOptionToken),
ISymbol(1, SymbolOptionToken)
})));
});
it("includes FIRST(A's right hand side) when A is a non-terminal", [&]() {
auto rule = choice({
seq({
sym("A"),
sym("x"),
sym("A") }),
sym("A") });
i_token(0),
i_token(1) }),
i_sym(0) });
Grammar grammar({
{ "A", choice({
seq({
sym("y"),
sym("z"),
sym("y") }),
sym("y") }) }
{ "rule0", seq({
i_token(2),
i_token(3),
i_token(4) }) }
});
AssertThat(first_set(rule, grammar), Equals(set<Symbol>({
Symbol("y")
AssertThat(first_set(rule, grammar), Equals(set<ISymbol>({
ISymbol(0, SymbolOptionToken),
ISymbol(2, SymbolOptionToken),
})));
});
it("includes FIRST(B) when A is a non-terminal and its expansion can be blank", [&]() {
Grammar grammar({{ "A", choice({ sym("x"), blank() }) }});
auto rule = seq({
sym("A"),
sym("y") });
i_sym(0),
i_token(1) });
Grammar grammar({
{ "rule0", choice({
i_token(0),
blank() }) }
});
AssertThat(first_set(rule, grammar), Equals(set<Symbol>({
Symbol("x"),
Symbol("y")
AssertThat(first_set(rule, grammar), Equals(set<ISymbol>({
ISymbol(0, SymbolOptionToken),
ISymbol(1, SymbolOptionToken),
})));
});
});
@ -73,23 +75,25 @@ describe("computing FIRST sets", []() {
describe("when there are left-recursive rules", [&]() {
it("terminates", [&]() {
Grammar grammar({
{ "expression", choice({
seq({ sym("expression"), sym("x") }),
sym("y"),
{ "rule0", choice({
seq({ i_sym(0), i_token(10) }),
i_token(11),
}) },
});
auto rule = i_sym(0);
AssertThat(first_set(sym("expression"), grammar), Equals(set<Symbol>({
Symbol("y")
AssertThat(first_set(rule, grammar), Equals(set<ISymbol>({
ISymbol(11, SymbolOptionToken)
})));
});
});
it("ignores metadata rules", [&]() {
auto rule = make_shared<Metadata>(sym("x"), map<rules::MetadataKey, int>());
auto rule = make_shared<Metadata>(i_token(3), map<rules::MetadataKey, int>());
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol("x"),
AssertThat(first_set(rule, null_grammar), Equals(set<ISymbol>({
ISymbol(3, SymbolOptionToken),
})));
});
});

44
spec/compiler/build_tables/follow_sets_spec.cc
View file

@ -12,38 +12,46 @@ describe("computing FOLLOW sets", []() {
const PreparedGrammar grammar({
{ "A", sym("a") },
{ "B", sym("b") },
{ "C", choice({ i_sym(0), i_sym(1) }) },
}, {});
it("includes all of the starting non-terminals for the item, and their following terminals", [&]() {
ParseItem item(Symbol("C"), choice({
seq({ sym("A"), choice({ sym("x"), sym("y") }) }),
seq({ sym("B"), sym("z") }),
}), {}, Symbol("w"));
ParseItem item(ISymbol(2), choice({
seq({ i_sym(0), choice({ i_token(0), i_token(1) }) }),
seq({ i_sym(1), i_token(2) }),
}), 0, ISymbol(10, SymbolOptionToken));
AssertThat(follow_sets(item, grammar), Equals(map<Symbol, set<Symbol>>({
{ Symbol("A"), set<Symbol>({ Symbol("x"), Symbol("y") }) },
{ Symbol("B"), set<Symbol>({ Symbol("z") }) },
AssertThat(follow_sets(item, grammar), Equals(map<ISymbol, set<ISymbol>>({
{ ISymbol(0), set<ISymbol>({
ISymbol(0, SymbolOptionToken),
ISymbol(1, SymbolOptionToken) }) },
{ ISymbol(1), set<ISymbol>({
ISymbol(2, SymbolOptionToken) }) },
})));
});
it("does not include terminals at the beginning of the item", [&]() {
ParseItem item(Symbol("C"), choice({
seq({ sym("A"), choice({ sym("x"), sym("y") }) }),
seq({ sym("x"), sym("y") }),
}), {}, Symbol("w"));
ParseItem item(ISymbol(2), choice({
seq({ i_sym(0), choice({ i_token(0), i_token(1) }) }),
seq({ i_token(2), i_token(3) }),
}), 0, ISymbol(10, SymbolOptionToken));
AssertThat(follow_sets(item, grammar), Equals(map<Symbol, set<Symbol>>({
{ Symbol("A"), set<Symbol>({ Symbol("x"), Symbol("y") }) },
AssertThat(follow_sets(item, grammar), Equals(map<ISymbol, set<ISymbol>>({
{ ISymbol(0), set<ISymbol>({
ISymbol(0, SymbolOptionToken),
ISymbol(1, SymbolOptionToken) }) },
})));
});
it("includes the item's lookahead terminal if the rule after the non-terminal might be blank", [&]() {
ParseItem item(Symbol("C"), choice({
seq({ sym("A"), choice({ sym("x"), blank() }) }),
}), {}, Symbol("w"));
ParseItem item(ISymbol(2), choice({
seq({ i_sym(0), choice({ i_token(0), blank() }) }),
}), 0, ISymbol(10, SymbolOptionToken));
AssertThat(follow_sets(item, grammar), Equals(map<Symbol, set<Symbol>>({
{ Symbol("A"), set<Symbol>({ Symbol("x"), Symbol("w") }) },
AssertThat(follow_sets(item, grammar), Equals(map<ISymbol, set<ISymbol>>({
{ ISymbol(0), set<ISymbol>({
ISymbol(0, SymbolOptionToken),
ISymbol(10, SymbolOptionToken) }) },
})));
});
});

40
spec/compiler/build_tables/item_set_closure_spec.cc
View file

@ -10,42 +10,22 @@ START_TEST
describe("computing closures of item sets", []() {
PreparedGrammar grammar({
{ "E", choice({
seq({
sym("T"),
sym("+"),
sym("T") }),
sym("T") }) },
{ "T", choice({
seq({
sym("F"),
sym("*"),
sym("F") }),
sym("F") }) },
{ "F", choice({
sym("v"),
sym("n") }) }
{ "E", seq({
i_sym(1),
i_token(11) }) },
{ "T", seq({
i_token(12),
i_token(13) }) },
}, {});
it("computes the item set closure", [&]() {
it("adds items at the beginnings of referenced rules", [&]() {
ParseItemSet item_set = item_set_closure(ParseItemSet({
ParseItem(Symbol("E"), grammar.rule(Symbol("E")), 0, Symbol("__END__"))
ParseItem(ISymbol(0), grammar.rule(ISymbol(0)), 0, ISymbol(10, SymbolOptionToken))
}), grammar);
AssertThat(item_set, Equals(ParseItemSet({
ParseItem(Symbol("F"), grammar.rule(Symbol("F")), 0, Symbol("__END__")),
ParseItem(Symbol("F"), grammar.rule(Symbol("F")), 0, Symbol("+")),
ParseItem(Symbol("F"), grammar.rule(Symbol("F")), 0, Symbol("*")),
ParseItem(Symbol("T"), grammar.rule(Symbol("T")), 0, Symbol("__END__")),
ParseItem(Symbol("T"), grammar.rule(Symbol("T")), 0, Symbol("+")),
ParseItem(Symbol("E"), grammar.rule(Symbol("E")), 0, Symbol("__END__")),
})));
ParseItemSet next_item_set = sym_transitions(item_set, grammar)[Symbol("v")];
AssertThat(next_item_set, Equals(ParseItemSet({
ParseItem(Symbol("F"), rules::blank(), 1, Symbol("__END__")),
ParseItem(Symbol("F"), rules::blank(), 1, Symbol("*")),
ParseItem(Symbol("F"), rules::blank(), 1, Symbol("+")),
ParseItem(ISymbol(1), grammar.rule(ISymbol(1)), 0, ISymbol(11, SymbolOptionToken)),
ParseItem(ISymbol(0), grammar.rule(ISymbol(0)), 0, ISymbol(10, SymbolOptionToken)),
})));
});
});

12
spec/compiler/build_tables/item_set_transitions_spec.cc
View file

@ -13,17 +13,17 @@ describe("item set transitions", []() {
describe("when two items in the set have transitions on the same character", [&]() {
it("merges the transitions by computing the union of the two item sets", [&]() {
LexItemSet set1({
LexItem(Symbol("A"), pattern("[a-f]")),
LexItem(Symbol("B"), pattern("[e-x]")) });
LexItem(ISymbol(1), pattern("[a-f]")),
LexItem(ISymbol(2), pattern("[e-x]")) });
AssertThat(char_transitions(set1, grammar), Equals(map<CharacterSet, LexItemSet>({
{ CharacterSet({ {'a', 'd'} }), LexItemSet({
LexItem(Symbol("A"), blank()) }) },
LexItem(ISymbol(1), blank()) }) },
{ CharacterSet({ {'e', 'f'} }), LexItemSet({
LexItem(Symbol("A"), blank()),
LexItem(Symbol("B"), blank()) }) },
LexItem(ISymbol(1), blank()),
LexItem(ISymbol(2), blank()) }) },
{ CharacterSet({ {'g', 'x'} }), LexItemSet({
LexItem(Symbol("B"), blank()) }) },
LexItem(ISymbol(2), blank()) }) },
})));
});
});

2
spec/compiler/build_tables/lex_item_spec.cc
View file

@ -9,7 +9,7 @@ START_TEST
describe("lex items", []() {
describe("determining if an item is the start of a token", [&]() {
Symbol sym("x");
ISymbol sym(1);
rule_ptr token_start = make_shared<Metadata>(str("a"), map<MetadataKey, int>({
{ START_TOKEN, 1 }
}));

12
spec/compiler/build_tables/rule_can_be_blank_spec.cc
View file

@ -12,7 +12,7 @@ describe("checking if rules can be blank", [&]() {
rule_ptr rule;
it("returns false for basic rules", [&]() {
AssertThat(rule_can_be_blank(sym("x")), IsFalse());
AssertThat(rule_can_be_blank(i_sym(3)), IsFalse());
AssertThat(rule_can_be_blank(str("x")), IsFalse());
AssertThat(rule_can_be_blank(pattern("x")), IsFalse());
});
@ -58,20 +58,20 @@ describe("checking if rules can be blank", [&]() {
describe("checking recursively (by expanding non-terminals)", [&]() {
PreparedGrammar grammar({
{ "A", choice({
seq({ sym("A"), sym("x") }),
seq({ i_sym(0), i_token(11) }),
blank() }) },
{ "B", choice({
seq({ sym("B"), sym("y") }),
sym("z") }) },
seq({ i_sym(1), i_token(12) }),
i_token(13) }) },
}, {});
it("terminates for left-recursive rules that can be blank", [&]() {
rule = sym("A");
rule = i_sym(0);
AssertThat(rule_can_be_blank(rule, grammar), IsTrue());
});
it("terminates for left-recursive rules that can't be blank", [&]() {
rule = sym("B");
rule = i_sym(1);
AssertThat(rule_can_be_blank(rule, grammar), IsFalse());
});
});

56
spec/compiler/build_tables/rule_transitions_spec.cc
View file

@ -26,39 +26,39 @@ START_TEST
describe("rule transitions", []() {
it("handles symbols", [&]() {
AssertThat(
sym_transitions(sym("1")),
Equals(rule_map<Symbol>({
{ Symbol("1"), blank() }
sym_transitions(i_sym(1)),
Equals(rule_map<ISymbol>({
{ ISymbol(1), blank() }
})));
});
it("handles choices", [&]() {
AssertThat(
sym_transitions(choice({ sym("1"), sym("2") })),
Equals(rule_map<Symbol>({
{ Symbol("1"), blank() },
{ Symbol("2"), blank() }
sym_transitions(choice({ i_sym(1), i_sym(2) })),
Equals(rule_map<ISymbol>({
{ ISymbol(1), blank() },
{ ISymbol(2), blank() }
})));
});
it("handles sequences", [&]() {
AssertThat(
sym_transitions(seq({ sym("1"), sym("2") })),
Equals(rule_map<Symbol>({
{ Symbol("1"), sym("2") }
sym_transitions(seq({ i_sym(1), i_sym(2) })),
Equals(rule_map<ISymbol>({
{ ISymbol(1), i_sym(2) }
})));
});
it("handles long sequences", [&]() {
AssertThat(
sym_transitions(seq({
sym("1"),
sym("2"),
sym("3"),
sym("4")
i_sym(1),
i_sym(2),
i_sym(3),
i_sym(4)
})),
Equals(rule_map<Symbol>({
{ Symbol("1"), seq({ sym("2"), sym("3"), sym("4") }) }
Equals(rule_map<ISymbol>({
{ ISymbol(1), seq({ i_sym(2), i_sym(3), i_sym(4) }) }
})));
});
@ -66,15 +66,15 @@ describe("rule transitions", []() {
AssertThat(
sym_transitions(seq({
choice({
sym("1"),
i_sym(1),
blank(),
}),
seq({
sym("1"),
sym("2")
i_sym(1),
i_sym(2)
})
})), Equals(rule_map<Symbol>({
{ Symbol("1"), choice({ seq({ sym("1"), sym("2") }), sym("2"), }) }
})), Equals(rule_map<ISymbol>({
{ ISymbol(1), choice({ seq({ i_sym(1), i_sym(2) }), i_sym(2), }) }
})));
});
@ -82,10 +82,10 @@ describe("rule transitions", []() {
AssertThat(
sym_transitions(
choice({
seq({ sym("1"), sym("2") }),
seq({ sym("1"), sym("3") }) })),
Equals(rule_map<Symbol>({
{ Symbol("1"), choice({ sym("2"), sym("3") }) }
seq({ i_sym(1), i_sym(2) }),
seq({ i_sym(1), i_sym(3) }) })),
Equals(rule_map<ISymbol>({
{ ISymbol(1), choice({ i_sym(2), i_sym(3) }) }
})));
});
@ -189,11 +189,11 @@ describe("rule transitions", []() {
{ PRECEDENCE, 5 }
});
rule_ptr rule = make_shared<Metadata>(seq({ sym("x"), sym("y") }), metadata_value);
rule_ptr rule = make_shared<Metadata>(seq({ i_sym(1), i_sym(2) }), metadata_value);
AssertThat(
sym_transitions(rule),
Equals(rule_map<Symbol>({
{ Symbol("x"), make_shared<Metadata>(sym("y"), metadata_value)},
Equals(rule_map<ISymbol>({
{ ISymbol(1), make_shared<Metadata>(i_sym(2), metadata_value)},
})));
});

2
spec/compiler/compiler_spec_helper.h
View file

@ -4,7 +4,7 @@
#include "bandit/bandit.h"
#include "helpers/stream_methods.h"
#include "helpers/equals_pointer.h"
#include "helpers/character_set_helpers.h"
#include "helpers/rule_helpers.h"
#include "tree_sitter/compiler.h"
using namespace tree_sitter;

20
spec/compiler/helpers/character_set_helpers.cc
View file

@ -1,20 +0,0 @@
#include "character_set_helpers.h"
#include <memory>
namespace tree_sitter {
using std::make_shared;
using std::set;
namespace rules {
rule_ptr character(const set<CharacterRange> &ranges) {
return make_shared<CharacterSet>(ranges);
}
rule_ptr character(const set<CharacterRange> &ranges, bool sign) {
if (sign)
return character(ranges);
else
return CharacterSet(ranges).complement().copy();
}
}
}

37
spec/compiler/helpers/rule_helpers.cc Normal file
View file

@ -0,0 +1,37 @@
#include "rule_helpers.h"
#include <memory>
#include "compiler/rules/interned_symbol.h"
namespace tree_sitter {
using std::make_shared;
using std::set;
namespace rules {
rule_ptr character(const set<CharacterRange> &ranges) {
return make_shared<CharacterSet>(ranges);
}
rule_ptr character(const set<CharacterRange> &ranges, bool sign) {
if (sign)
return character(ranges);
else
return CharacterSet(ranges).complement().copy();
}
rule_ptr i_sym(size_t index) {
return make_shared<rules::ISymbol>(index);
}
rule_ptr i_aux_sym(size_t index) {
return make_shared<rules::ISymbol>(index, SymbolOptionAuxiliary);
}
rule_ptr i_token(size_t index) {
return make_shared<rules::ISymbol>(index, SymbolOptionToken);
}
rule_ptr i_aux_token(size_t index) {
return make_shared<rules::ISymbol>(index, SymbolOption(SymbolOptionAuxiliary|SymbolOptionToken));
}
}
}

4
spec/compiler/helpers/character_set_helpers.h → spec/compiler/helpers/rule_helpers.h
View file

@ -8,6 +8,10 @@ namespace tree_sitter {
namespace rules {
rule_ptr character(const std::set<CharacterRange> &ranges);
rule_ptr character(const std::set<CharacterRange> &ranges, bool sign);
rule_ptr i_sym(size_t index);
rule_ptr i_aux_sym(size_t index);
rule_ptr i_token(size_t index);
rule_ptr i_aux_token(size_t index);
}
}

58
spec/compiler/name_symbols/name_symbols_spec.cc
View file

@ -1,58 +0,0 @@
#include "compiler_spec_helper.h"
#include "compiler/name_symbols/name_symbols.h"
#include "compiler/prepared_grammar.h"
#include "compiler/rules/built_in_symbols.h"
using namespace rules;
using name_symbols::name_symbols;
START_TEST
describe("assigning user-visible names to symbols", [&]() {
PreparedGrammar syntactic_grammar({
{ "some_syntactic_symbol", seq({
make_shared<Symbol>("some_given_name", SymbolTypeNormal),
make_shared<Symbol>("some_generated_string_name", SymbolTypeAuxiliary),
make_shared<Symbol>("some_generated_pattern_name", SymbolTypeAuxiliary), }) },
}, {});
PreparedGrammar lexical_grammar({
{ "some_given_name", str("the-string") },
}, {
{ "some_generated_string_name", str("the-string") },
{ "some_generated_pattern_name", pattern("the-pattern") },
});
map<Symbol, string> result = name_symbols::name_symbols(syntactic_grammar, lexical_grammar);
describe("for symbols that are not in the lexical grammar (syntactic rules)", [&]() {
it("uses the symbol's normal name", [&]() {
auto symbol = Symbol("some_syntactic_symbol");
AssertThat(result[symbol], Equals("some_syntactic_symbol"));
});
});
describe("for symbols that are in the lexical grammar", [&]() {
it("uses symbols' normal names when they are given by the user", [&]() {
auto symbol = Symbol("some_given_name");
AssertThat(result[symbol], Equals("some_given_name"));
});
it("assigns names to string rules based on their string value", [&]() {
auto symbol = Symbol("some_generated_string_name", rules::SymbolTypeAuxiliary);
AssertThat(result[symbol], Equals("'the-string'"));
});
it("assigns names to pattern rules based on their pattern value", [&]() {
auto symbol = Symbol("some_generated_pattern_name", rules::SymbolTypeAuxiliary);
AssertThat(result[symbol], Equals("/the-pattern/"));
});
});
it("assigns names to the built-in symbols", [&]() {
AssertThat(result[rules::END_OF_INPUT()], Equals("EOF"));
AssertThat(result[rules::ERROR()], Equals("ERROR"));
});
});
END_TEST

97
spec/compiler/prepare_grammar/expand_repeats_spec.cc Normal file
View file

@ -0,0 +1,97 @@
#include "compiler_spec_helper.h"
#include "compiler/prepared_grammar.h"
#include "compiler/prepare_grammar/expand_repeats.h"
#include "compiler/rules/symbol.h"
START_TEST
using namespace rules;
using prepare_grammar::expand_repeats;
describe("expanding repeat rules in a grammar", []() {
it("replaces repeat rules with pairs of recursive rules", [&]() {
PreparedGrammar grammar({
{ "rule0", repeat(i_token(0)) },
}, {});
AssertThat(expand_repeats(grammar), Equals(PreparedGrammar({
{ "rule0", i_aux_sym(0) },
}, {
{ "rule0_repeat0", choice({
seq({
i_token(0),
i_aux_sym(0) }),
blank() }) },
})));
});
it("replaces repeats inside of sequences", [&]() {
PreparedGrammar grammar({
{ "rule0", seq({ i_token(10), repeat(i_token(11)) }) },
}, {});
AssertThat(expand_repeats(grammar), Equals(PreparedGrammar({
{ "rule0", seq({ i_token(10), i_aux_sym(0) }) },
}, {
{ "rule0_repeat0", choice({
seq({ i_token(11), i_aux_sym(0) }),
blank() }) },
})));
});
it("replaces repeats inside of choices", [&]() {
PreparedGrammar grammar({
{ "rule0", choice({ i_token(10), repeat(i_token(11)) }) },
}, {});
AssertThat(expand_repeats(grammar), Equals(PreparedGrammar({
{ "rule0", choice({ i_token(10), i_aux_sym(0) }) },
}, {
{ "rule0_repeat0", choice({
seq({ i_token(11), i_aux_sym(0) }),
blank() }) },
})));
});
it("can replace multiple repeats in the same rule", [&]() {
PreparedGrammar grammar({
{ "rule0", seq({ repeat(i_token(10)), repeat(i_token(11)) }) },
}, {});
AssertThat(expand_repeats(grammar), Equals(PreparedGrammar({
{ "rule0", seq({ i_aux_sym(0), i_aux_sym(1) }) },
}, {
{ "rule0_repeat0", choice({
seq({
i_token(10),
i_aux_sym(0) }),
blank() }) },
{ "rule0_repeat1", choice({
seq({
i_token(11),
i_aux_sym(1) }),
blank() }) },
})));
});
it("can replace repeats in multiple rules", [&]() {
PreparedGrammar grammar({
{ "rule0", repeat(i_token(10)) },
{ "rule1", repeat(i_token(11)) },
}, {});
AssertThat(expand_repeats(grammar), Equals(PreparedGrammar({
{ "rule0", i_aux_sym(0) },
{ "rule1", i_aux_sym(1) },
}, {
{ "rule0_repeat0", choice({
seq({ i_token(10), i_aux_sym(0) }),
blank() }) },
{ "rule1_repeat0", choice({
seq({ i_token(11), i_aux_sym(1) }),
blank() }) },
})));
});
});
END_TEST

99
spec/compiler/prepare_grammar/extract_tokens_spec.cc Normal file
View file

@ -0,0 +1,99 @@
#include "compiler_spec_helper.h"
#include "compiler/prepared_grammar.h"
#include "compiler/prepare_grammar/extract_tokens.h"
#include "compiler/rules/symbol.h"
START_TEST
using namespace rules;
using prepare_grammar::extract_tokens;
describe("extracting tokens from a grammar", []() {
it("moves strings into the lexical grammar", [&]() {
pair<PreparedGrammar, PreparedGrammar> result = extract_tokens(PreparedGrammar({
{ "rule0", seq({ str("ab"), i_sym(0) }) }
}, {}));
AssertThat(result.first, Equals(PreparedGrammar({
{ "rule0", seq({ i_aux_token(0), i_sym(0) }) }
}, {})));
AssertThat(result.second, Equals(PreparedGrammar({}, {
{ "token0", str("ab") },
})));
});
it("moves patterns into the lexical grammar", [&]() {
pair<PreparedGrammar, PreparedGrammar> result = extract_tokens(PreparedGrammar({
{ "rule0", seq({ pattern("a+"), i_sym(0) }) }
}, {}));
AssertThat(result.first, Equals(PreparedGrammar({
{ "rule0", seq({ i_aux_token(0), i_sym(0) }) }
}, {})));
AssertThat(result.second, Equals(PreparedGrammar({}, {
{ "token0", pattern("a+") },
})));
});
it("does not extract blanks into tokens", [&]() {
pair<PreparedGrammar, PreparedGrammar> result = extract_tokens(Grammar({
{ "rule1", choice({ i_sym(0), blank() }) },
}));
AssertThat(result.first, Equals(PreparedGrammar({
{ "rule1", choice({ i_sym(0), blank() }) },
}, {})));
AssertThat(result.second, Equals(PreparedGrammar({}, {})));
});
it("does not create duplicate tokens in the lexical grammar", [&]() {
pair<PreparedGrammar, PreparedGrammar> result = extract_tokens(PreparedGrammar({
{ "rule0", seq({ str("ab"), i_sym(0), str("ab") }) },
}, {}));
AssertThat(result.first, Equals(PreparedGrammar({
{ "rule0", seq({ i_aux_token(0), i_sym(0), i_aux_token(0) }) }
}, {})));
AssertThat(result.second, Equals(PreparedGrammar({}, {
{ "token0", str("ab") },
})));
});
it("moves entire rules into the lexical grammar when possible, updating referencing symbols", [&]() {
auto result = extract_tokens(PreparedGrammar({
{ "rule0", i_sym(1) },
{ "rule1", pattern("a|b") },
}, {}));
AssertThat(result.first, Equals(PreparedGrammar({
{ "rule0", i_token(0) }
}, {})));
AssertThat(result.second, Equals(PreparedGrammar({
{ "rule1", pattern("a|b") },
}, {})));
});
it("updates symbols whose indices need to change due to deleted rules", [&]() {
auto result = extract_tokens(PreparedGrammar({
{ "rule0", str("ab") },
{ "rule1", i_sym(0) },
{ "rule2", i_sym(1) },
}, {}));
AssertThat(result.first, Equals(PreparedGrammar({
{ "rule1", i_token(0) },
{ "rule2", i_sym(0) },
}, {})));
AssertThat(result.second, Equals(PreparedGrammar({
{ "rule0", str("ab") },
}, {})));
});
});
END_TEST

43
spec/compiler/prepare_grammar/intern_symbols_spec.cc Normal file
View file

@ -0,0 +1,43 @@
#include "compiler_spec_helper.h"
#include "compiler/prepared_grammar.h"
#include "compiler/prepare_grammar/intern_symbols.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/interned_symbol.h"
START_TEST
using namespace rules;
using prepare_grammar::intern_symbols;
describe("interning symbols in a grammar", []() {
it("replaces named symbols with numerically-indexed symbols", [&]() {
Grammar grammar({
{ "x", choice({ sym("y"), sym("z") }) },
{ "y", sym("z") },
{ "z", str("stuff") }
});
auto result = intern_symbols(grammar);
AssertThat((bool)result.second, IsFalse());
AssertThat(result.first, Equals(PreparedGrammar({
{ "x", choice({ i_sym(1), i_sym(2) }) },
{ "y", i_sym(2) },
{ "z", str("stuff") },
}, {})));
});
describe("when there are symbols that reference undefined rules", [&]() {
it("returns an error", []() {
Grammar grammar({
{ "x", sym("y") },
});
auto result = intern_symbols(grammar);
AssertThat(result.second->message(), Equals("Undefined rule 'y'"));
});
});
});
END_TEST

106
spec/compiler/prepare_grammar_spec.cc
View file

@ -1,106 +0,0 @@
#include "compiler_spec_helper.h"
#include "compiler/prepared_grammar.h"
#include "compiler/prepare_grammar/prepare_grammar.h"
#include "compiler/rules/symbol.h"
START_TEST
using namespace rules;
using prepare_grammar::prepare_grammar;
describe("preparing a grammar", []() {
describe("extracting tokens", []() {
it("moves strings and patterns into a separate 'lexical' grammar", [&]() {
pair<PreparedGrammar, PreparedGrammar> result = prepare_grammar(Grammar({
{ "rule1", seq({
str("ab"),
seq({
sym("rule2"),
sym("rule3") }),
str("ab") }) }
}));
AssertThat(result.first, Equals(PreparedGrammar({
{ "rule1", seq({
make_shared<Symbol>("token1", SymbolTypeAuxiliary),
seq({
sym("rule2"),
sym("rule3") }),
make_shared<Symbol>("token1", SymbolTypeAuxiliary) }) }
}, {})));
AssertThat(result.second, Equals(PreparedGrammar({}, {
{ "token1", str("ab") },
})));
});
it("moves entire rules into the lexical grammar when possible, preserving their names", [&]() {
auto result = prepare_grammar(Grammar({
{ "rule1", sym("rule2") },
{ "rule2", pattern("a|b") }
}));
AssertThat(result.first, Equals(PreparedGrammar({
{ "rule1", sym("rule2") }
}, {})));
AssertThat(result.second, Equals(PreparedGrammar({
{ "rule2", pattern("a|b") },
}, {})));
});
it("does not extract blanks into tokens", [&]() {
pair<PreparedGrammar, PreparedGrammar> result = prepare_grammar(Grammar({
{ "rule1", choice({ sym("rule2"), blank() }) },
}));
AssertThat(result.first, Equals(PreparedGrammar({
{ "rule1", choice({ sym("rule2"), blank() }) },
}, {})));
AssertThat(result.second, Equals(PreparedGrammar({}, {})));
});
});
describe("expanding repeats", []() {
it("replaces repeat rules with pairs of recursive rules", [&]() {
PreparedGrammar result = prepare_grammar(Grammar({
{ "rule1", seq({
sym("x"),
repeat(seq({ sym("a"), sym("b") })),
repeat(sym("a")),
sym("y")
}) },
{ "rule2", repeat(sym("b")) },
})).first;
AssertThat(result, Equals(PreparedGrammar({
{ "rule1", seq({
sym("x"),
make_shared<Symbol>("rule1_repeat1", SymbolTypeAuxiliary),
make_shared<Symbol>("rule1_repeat2", SymbolTypeAuxiliary),
sym("y") }) },
{ "rule2", make_shared<Symbol>("rule2_repeat1", SymbolTypeAuxiliary) },
}, {
{ "rule1_repeat1", choice({
seq({
seq({ sym("a"), sym("b") }),
make_shared<Symbol>("rule1_repeat1", SymbolTypeAuxiliary),
}),
blank() }) },
{ "rule1_repeat2", choice({
seq({
sym("a"),
make_shared<Symbol>("rule1_repeat2", SymbolTypeAuxiliary) }),
blank() }) },
{ "rule2_repeat1", choice({
seq({
sym("b"),
make_shared<Symbol>("rule2_repeat1", SymbolTypeAuxiliary) }),
blank() }) }
})));
});
});
});
END_TEST

27
src/compiler/build_tables/build_tables.cc
View file

@ -10,6 +10,7 @@
#include "compiler/rules/repeat.h"
#include "compiler/rules/blank.h"
#include "compiler/rules/seq.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/build_tables/conflict_manager.h"
#include "compiler/build_tables/item.h"
#include "compiler/build_tables/item_set_closure.h"
@ -24,7 +25,7 @@ namespace tree_sitter {
using std::set;
using std::unordered_map;
using std::make_shared;
using rules::Symbol;
using rules::ISymbol;
using rules::CharacterSet;
namespace build_tables {
@ -45,7 +46,7 @@ namespace tree_sitter {
void add_shift_actions(const ParseItemSet &item_set, ParseStateId state_id) {
for (auto &transition : sym_transitions(item_set, grammar)) {
const Symbol &symbol = transition.first;
const ISymbol &symbol = transition.first;
const ParseItemSet &item_set = transition.second;
set<int> precedence_values = precedence_values_for_item_set(item_set);
@ -117,7 +118,7 @@ namespace tree_sitter {
LexItemSet lex_item_set_for_parse_state(const ParseState &state) {
LexItemSet result;
for (auto &symbol : state.expected_inputs()) {
if (lex_grammar.has_definition(symbol))
if (symbol.is_token() && !symbol.is_built_in())
result.insert(LexItem(symbol, after_separators(lex_grammar.rule(symbol))));
if (symbol == rules::END_OF_INPUT())
result.insert(LexItem(symbol, after_separators(CharacterSet({ 0 }).copy())));
@ -160,12 +161,12 @@ namespace tree_sitter {
void add_error_lex_state() {
LexItemSet error_item_set;
for (auto &pair : lex_grammar.rules) {
LexItem item(Symbol(pair.first, rules::SymbolTypeNormal), after_separators(pair.second));
for (size_t i = 0; i < lex_grammar.rules.size(); i++) {
LexItem item(ISymbol(i, rules::SymbolOptionToken), after_separators(lex_grammar.rules[i].second));
error_item_set.insert(item);
}
for (auto &pair : lex_grammar.aux_rules) {
LexItem item(Symbol(pair.first, rules::SymbolTypeAuxiliary), after_separators(pair.second));
for (size_t i = 0; i < lex_grammar.aux_rules.size(); i++) {
LexItem item(ISymbol(i, rules::SymbolOption(rules::SymbolOptionToken|rules::SymbolOptionAuxiliary)), after_separators(lex_grammar.aux_rules[i].second));
error_item_set.insert(item);
}
error_item_set.insert(LexItem(rules::END_OF_INPUT(), after_separators(CharacterSet({ 0 }).copy())));
@ -175,15 +176,14 @@ namespace tree_sitter {
public:
TableBuilder(const PreparedGrammar &grammar,
const PreparedGrammar &lex_grammar,
const map<Symbol, string> &rule_names) :
const PreparedGrammar &lex_grammar) :
grammar(grammar),
lex_grammar(lex_grammar),
conflict_manager(ConflictManager(grammar, lex_grammar, rule_names))
conflict_manager(ConflictManager(grammar, lex_grammar))
{}
void build() {
auto start_symbol = make_shared<Symbol>(grammar.start_rule_name());
auto start_symbol = make_shared<ISymbol>(0);
ParseItem item(rules::START(), start_symbol, {}, rules::END_OF_INPUT());
ParseItemSet item_set = item_set_closure(ParseItemSet({ item }), grammar);
add_parse_state(item_set);
@ -200,9 +200,8 @@ namespace tree_sitter {
pair<pair<ParseTable, LexTable>, vector<Conflict>>
build_tables(const PreparedGrammar &grammar,
const PreparedGrammar &lex_grammar,
const map<Symbol, string> &rule_names) {
TableBuilder builder(grammar, lex_grammar, rule_names);
const PreparedGrammar &lex_grammar) {
TableBuilder builder(grammar, lex_grammar);
builder.build();
return { { builder.parse_table, builder.lex_table }, builder.conflicts() };
}

3
src/compiler/build_tables/build_tables.h
View file

@ -15,8 +15,7 @@ namespace tree_sitter {
namespace build_tables {
std::pair<std::pair<ParseTable, LexTable>, std::vector<Conflict>>
build_tables(const PreparedGrammar &grammar,
const PreparedGrammar &lex_grammar,
const std::map<rules::Symbol, std::string> &rule_names);
const PreparedGrammar &lex_grammar);
}
}

38
src/compiler/build_tables/conflict_manager.cc
View file

@ -15,14 +15,11 @@ namespace tree_sitter {
using std::vector;
ConflictManager::ConflictManager(const PreparedGrammar &parse_grammar,
const PreparedGrammar &lex_grammar,
const map<Symbol, string> &rule_names) :
const PreparedGrammar &lex_grammar) :
parse_grammar(parse_grammar),
lex_grammar(lex_grammar),
rule_names(rule_names)
{}
lex_grammar(lex_grammar) {}
bool ConflictManager::resolve_parse_action(const rules::Symbol &symbol,
bool ConflictManager::resolve_parse_action(const rules::ISymbol &symbol,
const ParseAction &old_action,
const ParseAction &new_action) {
if (new_action.type < old_action.type)
@ -63,9 +60,7 @@ namespace tree_sitter {
return false;
} else {
record_conflict(symbol, old_action, new_action);
size_t old_index = parse_grammar.index_of(old_action.symbol);
size_t new_index = parse_grammar.index_of(new_action.symbol);
return new_index < old_index;
return new_action.symbol.index < old_action.symbol.index;
}
}
default:
@ -83,9 +78,7 @@ namespace tree_sitter {
return true;
case LexActionTypeAccept:
if (new_action.type == LexActionTypeAccept) {
size_t old_index = lex_grammar.index_of(old_action.symbol);
size_t new_index = lex_grammar.index_of(new_action.symbol);
return (new_index < old_index);
return new_action.symbol.index < old_action.symbol.index;
}
default:
return false;
@ -109,16 +102,16 @@ namespace tree_sitter {
return precedences + ")";
}
string message_for_action(const ParseAction &action, const map<Symbol, string> &rule_names) {
string message_for_action(const ParseAction &action, const PreparedGrammar &parse_grammar) {
switch (action.type) {
case ParseActionTypeShift:
return "shift " + precedence_string(action);
case ParseActionTypeReduce: {
auto pair = rule_names.find(action.symbol);
if (pair == rule_names.end())
return "ERROR " + action.symbol.name;
string name = parse_grammar.rule_name(action.symbol);
if (name == "")
return "ERROR" + to_string(action.symbol.index);
else
return "reduce " + pair->second + " " + precedence_string(action);
return "reduce " + name + " " + precedence_string(action);
}
case ParseActionTypeAccept:
return "accept";
@ -127,12 +120,15 @@ namespace tree_sitter {
}
}
void ConflictManager::record_conflict(const rules::Symbol &symbol,
void ConflictManager::record_conflict(const rules::ISymbol &symbol,
const ParseAction &left,
const ParseAction &right) {
conflicts_.insert(Conflict(rule_names.find(symbol)->second + ": " +
message_for_action(left, rule_names) + " / " +
message_for_action(right, rule_names)));
string name = symbol.is_token() ?
lex_grammar.rule_name(symbol) :
parse_grammar.rule_name(symbol);
conflicts_.insert(Conflict(name + ": " +
message_for_action(left, parse_grammar) + " / " +
message_for_action(right, parse_grammar)));
}
}
}

9
src/compiler/build_tables/conflict_manager.h
View file

@ -7,6 +7,7 @@
#include <set>
#include "tree_sitter/compiler.h"
#include "compiler/parse_table.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/prepared_grammar.h"
namespace tree_sitter {
@ -14,21 +15,19 @@ namespace tree_sitter {
class ConflictManager {
const PreparedGrammar parse_grammar;
const PreparedGrammar lex_grammar;
const std::map<rules::Symbol, std::string> rule_names;
std::set<Conflict> conflicts_;
public:
ConflictManager(const PreparedGrammar &parse_grammar,
const PreparedGrammar &lex_grammar,
const std::map<rules::Symbol, std::string> &rule_names);
const PreparedGrammar &lex_grammar);
bool resolve_lex_action(const LexAction &old_action,
const LexAction &new_action);
bool resolve_parse_action(const rules::Symbol &symbol,
bool resolve_parse_action(const rules::ISymbol &symbol,
const ParseAction &old_action,
const ParseAction &new_action);
void record_conflict(const rules::Symbol &symbol, const ParseAction &left, const ParseAction &right);
void record_conflict(const rules::ISymbol &symbol, const ParseAction &left, const ParseAction &right);
const std::vector<Conflict> conflicts() const;
};
}

33
src/compiler/build_tables/first_set.cc
View file

@ -6,47 +6,48 @@
#include "compiler/rules/visitor.h"
#include "compiler/rules/seq.h"
#include "compiler/rules/choice.h"
#include "compiler/rules/interned_symbol.h"
namespace tree_sitter {
using std::set;
using rules::Symbol;
using rules::ISymbol;
namespace build_tables {
set<Symbol> set_union(const set<Symbol> &left, const set<Symbol> &right) {
set<Symbol> result = left;
set<ISymbol> set_union(const set<ISymbol> &left, const set<ISymbol> &right) {
set<ISymbol> result = left;
result.insert(right.begin(), right.end());
return result;
}
class FirstSet : public rules::RuleFn<set<Symbol>> {
class FirstSet : public rules::RuleFn<set<ISymbol>> {
const PreparedGrammar grammar;
set<Symbol> visited_symbols;
set<ISymbol> visited_symbols;
public:
explicit FirstSet(const PreparedGrammar &grammar) : grammar(grammar) {}
set<Symbol> apply_to(const Symbol *rule) {
set<ISymbol> apply_to(const ISymbol *rule) {
if (visited_symbols.find(*rule) == visited_symbols.end()) {
visited_symbols.insert(*rule);
if (grammar.has_definition(*rule)) {
return apply(grammar.rule(*rule));
if (rule->is_token()) {
return set<ISymbol>({ *rule });
} else {
return set<Symbol>({ *rule });
return apply(grammar.rule(*rule));
}
} else {
return set<Symbol>();
return set<ISymbol>();
}
}
set<Symbol> apply_to(const rules::Metadata *rule) {
set<ISymbol> apply_to(const rules::Metadata *rule) {
return apply(rule->rule);
}
set<Symbol> apply_to(const rules::Choice *rule) {
set<ISymbol> apply_to(const rules::Choice *rule) {
return set_union(apply(rule->left), apply(rule->right));
}
set<Symbol> apply_to(const rules::Seq *rule) {
set<ISymbol> apply_to(const rules::Seq *rule) {
auto result = apply(rule->left);
if (rule_can_be_blank(rule->left, grammar)) {
return set_union(result, apply(rule->right));
@ -56,12 +57,12 @@ namespace tree_sitter {
}
};
set<Symbol> first_set(const rules::rule_ptr &rule, const PreparedGrammar &grammar) {
set<ISymbol> first_set(const rules::rule_ptr &rule, const PreparedGrammar &grammar) {
return FirstSet(grammar).apply(rule);
}
set<Symbol> first_set(const ParseItemSet &item_set, const PreparedGrammar &grammar) {
set<Symbol> result;
set<ISymbol> first_set(const ParseItemSet &item_set, const PreparedGrammar &grammar) {
set<ISymbol> result;
for (auto &item : item_set) {
result = set_union(result, first_set(item.rule, grammar));
if (rule_can_be_blank(item.rule, grammar))

4
src/compiler/build_tables/first_set.h
View file

@ -15,14 +15,14 @@ namespace tree_sitter {
* the beginning of a string derivable from a given rule,
* in a given gramamr.
*/
std::set<rules::Symbol>
std::set<rules::ISymbol>
first_set(const rules::rule_ptr &rule, const PreparedGrammar &grammar);
/*
* Returns the set of terminal symbols that can appear at
* the beginning of any item in the given set.
*/
std::set<rules::Symbol>
std::set<rules::ISymbol>
first_set(const ParseItemSet &item_set, const PreparedGrammar &grammar);
}
}

12
src/compiler/build_tables/follow_sets.cc
View file

@ -7,19 +7,19 @@
namespace tree_sitter {
using std::set;
using std::map;
using rules::Symbol;
using rules::ISymbol;
using rules::rule_ptr;
namespace build_tables {
map<Symbol, set<Symbol>> follow_sets(const ParseItem &item,
map<ISymbol, set<ISymbol>> follow_sets(const ParseItem &item,
const PreparedGrammar &grammar) {
map<Symbol, set<Symbol>> result;
map<ISymbol, set<ISymbol>> result;
for (auto &pair : sym_transitions(item.rule)) {
Symbol symbol = pair.first;
ISymbol symbol = pair.first;
rule_ptr next_rule = pair.second;
if (grammar.has_definition(symbol)) {
set<Symbol> following_terminals = first_set(next_rule, grammar);
if (!symbol.is_token() && !symbol.is_built_in()) {
set<ISymbol> following_terminals = first_set(next_rule, grammar);
if (rule_can_be_blank(next_rule, grammar))
following_terminals.insert(item.lookahead_sym);
result.insert({ symbol, following_terminals });

2
src/compiler/build_tables/follow_sets.h
View file

@ -17,7 +17,7 @@ namespace tree_sitter {
* item. The values are the sets of terminals which can appear immediately
* after the corresponding non-terminals.
*/
std::map<rules::Symbol, std::set<rules::Symbol>>
std::map<rules::ISymbol, std::set<rules::ISymbol>>
follow_sets(const ParseItem &item, const PreparedGrammar &grammar);
}

2
src/compiler/build_tables/item.cc
View file

@ -4,7 +4,7 @@
namespace tree_sitter {
namespace build_tables {
Item::Item(const rules::Symbol &lhs, const rules::rule_ptr rule) :
Item::Item(const rules::ISymbol &lhs, const rules::rule_ptr rule) :
lhs(lhs),
rule(rule) {}

6
src/compiler/build_tables/item.h
View file

@ -2,16 +2,16 @@
#define COMPILER_BUILD_TABLES_ITEM_H_
#include <unordered_set>
#include "compiler/rules/symbol.h"
#include "compiler/rules/interned_symbol.h"
namespace tree_sitter {
namespace build_tables {
class Item {
public:
Item(const rules::Symbol &lhs, rules::rule_ptr rule);
Item(const rules::ISymbol &lhs, rules::rule_ptr rule);
bool is_done() const;
rules::Symbol lhs;
rules::ISymbol lhs;
rules::rule_ptr rule;
};
}

6
src/compiler/build_tables/item_set_closure.cc
View file

@ -8,7 +8,7 @@
namespace tree_sitter {
using std::set;
using rules::Symbol;
using rules::ISymbol;
namespace build_tables {
static bool contains(const ParseItemSet *items, const ParseItem &item) {
@ -22,8 +22,8 @@ namespace tree_sitter {
if (!contains(item_set, item)) {
item_set->insert(item);
for (const auto &pair : follow_sets(item, grammar)) {
const Symbol &non_terminal = pair.first;
const set<Symbol> &terminals = pair.second;
const ISymbol &non_terminal = pair.first;
const set<ISymbol> &terminals = pair.second;
for (const auto &terminal : terminals) {
ParseItem next_item(non_terminal, grammar.rule(non_terminal), 0, terminal);
add_item(item_set, next_item, grammar);

15
src/compiler/build_tables/item_set_transitions.cc
View file

@ -3,12 +3,13 @@
#include "compiler/build_tables/item_set_closure.h"
#include "compiler/build_tables/rule_transitions.h"
#include "compiler/build_tables/merge_transitions.h"
#include "compiler/rules/interned_symbol.h"
namespace tree_sitter {
using std::map;
using std::unordered_set;
using rules::CharacterSet;
using rules::Symbol;
using rules::ISymbol;
namespace build_tables {
map<CharacterSet, LexItemSet>
@ -21,11 +22,11 @@ namespace tree_sitter {
return result;
}
map<Symbol, ParseItemSet>
map<ISymbol, ParseItemSet>
sym_transitions(const ParseItem &item, const PreparedGrammar &grammar) {
map<Symbol, ParseItemSet> result;
map<ISymbol, ParseItemSet> result;
for (auto transition : sym_transitions(item.rule)) {
Symbol rule = transition.first;
ISymbol rule = transition.first;
ParseItem new_item(item.lhs, transition.second, item.consumed_symbol_count + 1, item.lookahead_sym);
result.insert({ rule, item_set_closure(ParseItemSet({ new_item }), grammar) });
}
@ -53,11 +54,11 @@ namespace tree_sitter {
return result;
}
map<Symbol, ParseItemSet>
map<ISymbol, ParseItemSet>
sym_transitions(const ParseItemSet &item_set, const PreparedGrammar &grammar) {
map<Symbol, ParseItemSet> result;
map<ISymbol, ParseItemSet> result;
for (const ParseItem &item : item_set) {
map<Symbol, ParseItemSet> item_transitions = sym_transitions(item, grammar);
map<ISymbol, ParseItemSet> item_transitions = sym_transitions(item, grammar);
result = merge_sym_transitions<ParseItemSet>(result,
item_transitions,
[&](ParseItemSet left, ParseItemSet right) {

2
src/compiler/build_tables/item_set_transitions.h
View file

@ -16,7 +16,7 @@ namespace tree_sitter {
std::map<rules::CharacterSet, LexItemSet>
char_transitions(const LexItemSet &item_set, const PreparedGrammar &grammar);
std::map<rules::Symbol, ParseItemSet>
std::map<rules::ISymbol, ParseItemSet>
sym_transitions(const ParseItemSet &item_set, const PreparedGrammar &grammar);
}
}

2
src/compiler/build_tables/lex_item.cc
View file

@ -11,7 +11,7 @@ namespace tree_sitter {
using std::vector;
namespace build_tables {
LexItem::LexItem(const rules::Symbol &lhs, const rules::rule_ptr rule) :
LexItem::LexItem(const rules::ISymbol &lhs, const rules::rule_ptr rule) :
Item(lhs, rule) {}
bool LexItem::operator==(const LexItem &other) const {

4
src/compiler/build_tables/lex_item.h
View file

@ -9,7 +9,7 @@ namespace tree_sitter {
namespace build_tables {
class LexItem : public Item {
public:
LexItem(const rules::Symbol &lhs, rules::rule_ptr rule);
LexItem(const rules::ISymbol &lhs, rules::rule_ptr rule);
bool operator==(const LexItem &other) const;
bool is_token_start() const;
};
@ -25,7 +25,7 @@ namespace std {
struct hash<tree_sitter::build_tables::LexItem> {
size_t operator()(const tree_sitter::build_tables::Item &item) const {
return
hash<tree_sitter::rules::Symbol>()(item.lhs) ^
hash<tree_sitter::rules::ISymbol>()(item.lhs) ^
hash<tree_sitter::rules::rule_ptr>()(item.rule);
}
};

10
src/compiler/build_tables/merge_transitions.h
View file

@ -3,7 +3,7 @@
#include <map>
#include "compiler/rules/character_set.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/interned_symbol.h"
namespace tree_sitter {
namespace build_tables {
@ -15,11 +15,11 @@ namespace tree_sitter {
* using the given function.
*/
template<typename T>
std::map<rules::Symbol, T>
merge_sym_transitions(const std::map<rules::Symbol, T> &left,
const std::map<rules::Symbol, T> &right,
std::map<rules::ISymbol, T>
merge_sym_transitions(const std::map<rules::ISymbol, T> &left,
const std::map<rules::ISymbol, T> &right,
std::function<T(T, T)> merge_fn) {
std::map<rules::Symbol, T> result(left);
std::map<rules::ISymbol, T> result(left);
for (auto &pair : right) {
auto rule = pair.first;
bool merged = false;

4
src/compiler/build_tables/parse_item.cc
View file

@ -8,10 +8,10 @@ namespace tree_sitter {
using std::ostream;
namespace build_tables {
ParseItem::ParseItem(const rules::Symbol &lhs,
ParseItem::ParseItem(const rules::ISymbol &lhs,
const rules::rule_ptr rule,
size_t consumed_symbol_count,
const rules::Symbol &lookahead_sym) :
const rules::ISymbol &lookahead_sym) :
Item(lhs, rule),
consumed_symbol_count(consumed_symbol_count),
lookahead_sym(lookahead_sym) {}

12
src/compiler/build_tables/parse_item.h
View file

@ -3,7 +3,7 @@
#include <unordered_set>
#include <string>
#include "compiler/rules/symbol.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/build_tables/item.h"
#include "compiler/rules/metadata.h"
@ -11,15 +11,15 @@ namespace tree_sitter {
namespace build_tables {
class ParseItem : public Item {
public:
ParseItem(const rules::Symbol &lhs,
ParseItem(const rules::ISymbol &lhs,
rules::rule_ptr rule,
const size_t consumed_symbol_count,
const rules::Symbol &lookahead_sym);
const rules::ISymbol &lookahead_sym);
bool operator==(const ParseItem &other) const;
int precedence() const;
const size_t consumed_symbol_count;
const rules::Symbol lookahead_sym;
const rules::ISymbol lookahead_sym;
};
std::ostream& operator<<(std::ostream &stream, const ParseItem &item);
@ -33,10 +33,10 @@ namespace std {
struct hash<tree_sitter::build_tables::ParseItem> {
size_t operator()(const tree_sitter::build_tables::ParseItem &item) const {
return
hash<string>()(item.lhs.name) ^
hash<tree_sitter::rules::ISymbol>()(item.lhs) ^
hash<tree_sitter::rules::rule_ptr>()(item.rule) ^
hash<size_t>()(item.consumed_symbol_count) ^
hash<string>()(item.lookahead_sym.name);
hash<tree_sitter::rules::ISymbol>()(item.lookahead_sym);
}
};

6
src/compiler/build_tables/rule_can_be_blank.cc
View file

@ -38,17 +38,17 @@ namespace tree_sitter {
class CanBeBlankRecursive : public CanBeBlank {
const PreparedGrammar grammar;
set<rules::Symbol> visited_symbols;
set<rules::ISymbol> visited_symbols;
using CanBeBlank::visit;
public:
using CanBeBlank::apply_to;
explicit CanBeBlankRecursive(const PreparedGrammar &grammar) : grammar(grammar) {}
bool apply_to(const rules::Symbol *rule) {
bool apply_to(const rules::ISymbol *rule) {
if (visited_symbols.find(*rule) == visited_symbols.end()) {
visited_symbols.insert(*rule);
return grammar.has_definition(*rule) && apply(grammar.rule(*rule));
return !rule->is_token() && apply(grammar.rule(*rule));
} else {
return false;
}

13
src/compiler/build_tables/rule_transitions.cc
View file

@ -8,6 +8,7 @@
#include "compiler/rules/string.h"
#include "compiler/rules/repeat.h"
#include "compiler/rules/metadata.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/pattern.h"
#include "compiler/rules/character_set.h"
#include "compiler/rules/visitor.h"
@ -17,7 +18,7 @@ namespace tree_sitter {
using std::set;
using std::make_shared;
using rules::rule_ptr;
using rules::Symbol;
using rules::ISymbol;
using rules::CharacterSet;
using rules::Metadata;
@ -35,8 +36,8 @@ namespace tree_sitter {
}
template<>
map<Symbol, rule_ptr>
merge_transitions(const map<Symbol, rule_ptr> &left, const map<Symbol, rule_ptr> &right) {
map<ISymbol, rule_ptr>
merge_transitions(const map<ISymbol, rule_ptr> &left, const map<ISymbol, rule_ptr> &right) {
return merge_sym_transitions<rule_ptr>(left, right, [](rule_ptr left, rule_ptr right) {
return make_shared<rules::Choice>(left, right);
});
@ -65,7 +66,7 @@ namespace tree_sitter {
return apply_to_atom(rule);
}
map<T, rule_ptr> apply_to(const Symbol *rule) {
map<T, rule_ptr> apply_to(const ISymbol *rule) {
return apply_to_atom(rule);
}
@ -117,8 +118,8 @@ namespace tree_sitter {
return RuleTransitions<CharacterSet>().apply(rule);
}
map<Symbol, rule_ptr> sym_transitions(const rule_ptr &rule) {
return RuleTransitions<Symbol>().apply(rule);
map<ISymbol, rule_ptr> sym_transitions(const rule_ptr &rule) {
return RuleTransitions<ISymbol>().apply(rule);
}
}
}

4
src/compiler/build_tables/rule_transitions.h
View file

@ -3,14 +3,14 @@
#include <map>
#include "compiler/rules/character_set.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/interned_symbol.h"
namespace tree_sitter {
namespace build_tables {
std::map<rules::CharacterSet, rules::rule_ptr>
char_transitions(const rules::rule_ptr &rule);
std::map<rules::Symbol, rules::rule_ptr>
std::map<rules::ISymbol, rules::rule_ptr>
sym_transitions(const rules::rule_ptr &rule);
}
}

7
src/compiler/compile.cc
View file

@ -3,7 +3,6 @@
#include "compiler/build_tables/build_tables.h"
#include "compiler/generate_code/c_code.h"
#include "compiler/prepared_grammar.h"
#include "compiler/name_symbols/name_symbols.h"
namespace tree_sitter {
using std::pair;
@ -15,15 +14,13 @@ namespace tree_sitter {
PreparedGrammar &syntax_grammar = grammars.first;
PreparedGrammar &lexical_grammar = grammars.second;
auto symbol_names = name_symbols::name_symbols(syntax_grammar, lexical_grammar);
auto table_build_result = build_tables::build_tables(syntax_grammar, lexical_grammar, symbol_names);
auto table_build_result = build_tables::build_tables(syntax_grammar, lexical_grammar);
auto tables = table_build_result.first;
auto conflicts = table_build_result.second;
ParseTable &parse_table = tables.first;
LexTable &lex_table = tables.second;
return { generate_code::c_code(name, parse_table, lex_table, symbol_names), conflicts };
return { generate_code::c_code(name, parse_table, lex_table, syntax_grammar, lexical_grammar), conflicts };
}
}

49
src/compiler/generate_code/c_code.cc
View file

@ -6,6 +6,8 @@
#include "compiler/generate_code/c_code.h"
#include "compiler/util/string_helpers.h"
#include "compiler/rules/built_in_symbols.h"
#include "compiler/prepared_grammar.h"
#include "compiler/generate_code/token_description.h"
namespace tree_sitter {
using std::string;
@ -52,17 +54,19 @@ namespace tree_sitter {
const string name;
const ParseTable parse_table;
const LexTable lex_table;
const map<rules::Symbol, string> symbol_names;
const PreparedGrammar syntax_grammar;
const PreparedGrammar lexical_grammar;
public:
CCodeGenerator(string name,
const ParseTable &parse_table,
const LexTable &lex_table,
const map<rules::Symbol, string> &symbol_names) :
const PreparedGrammar &syntax_grammar,
const PreparedGrammar &lexical_grammar) :
name(name),
parse_table(parse_table),
lex_table(lex_table),
symbol_names(symbol_names)
{}
syntax_grammar(syntax_grammar),
lexical_grammar(lexical_grammar) {}
string code() {
return join({
@ -79,15 +83,33 @@ namespace tree_sitter {
}
private:
string symbol_id(rules::Symbol symbol) {
const PreparedGrammar & grammar_for_symbol(const rules::ISymbol &symbol) {
return symbol.is_token() ? lexical_grammar : syntax_grammar;
}
string symbol_id(const rules::ISymbol &symbol) {
if (symbol.is_built_in()) {
return (symbol == rules::ERROR()) ?
"ts_builtin_sym_error" :
"ts_builtin_sym_end";
} else if (symbol.is_auxiliary()) {
return "ts_aux_sym_" + symbol.name;
} else {
return "ts_sym_" + symbol.name;
string name = grammar_for_symbol(symbol).rule_name(symbol);
if (symbol.is_auxiliary())
return "ts_aux_sym_" + name;
else
return "ts_sym_" + name;
}
}
string symbol_name(const rules::ISymbol &symbol) {
if (symbol.is_built_in()) {
return (symbol == rules::ERROR()) ? "error" : "end";
} else if (symbol.is_token() && symbol.is_auxiliary()) {
return token_description(grammar_for_symbol(symbol).rule(symbol));
} else {
string name = grammar_for_symbol(symbol).rule_name(symbol);
return name;
}
}
@ -189,20 +211,20 @@ namespace tree_sitter {
}
string symbol_names_list() {
set<rules::Symbol> symbols(parse_table.symbols);
set<rules::ISymbol> symbols(parse_table.symbols);
symbols.insert(rules::END_OF_INPUT());
symbols.insert(rules::ERROR());
string result = "SYMBOL_NAMES = {\n";
for (auto symbol : parse_table.symbols)
result += indent("[" + symbol_id(symbol) + "] = \"" + symbol_names.find(symbol)->second) + "\",\n";
result += indent("[" + symbol_id(symbol) + "] = \"" + symbol_name(symbol)) + "\",\n";
return result + "};";
}
string hidden_symbols_list() {
string result = "HIDDEN_SYMBOLS = {\n";
for (auto &symbol : parse_table.symbols)
if (symbol.is_hidden())
if (!symbol.is_built_in() && (symbol.is_auxiliary() || grammar_for_symbol(symbol).rule_name(symbol)[0] == '_'))
result += indent("[" + symbol_id(symbol) + "] = 1,") + "\n";
return result + "};";
}
@ -266,8 +288,9 @@ namespace tree_sitter {
string c_code(string name,
const ParseTable &parse_table,
const LexTable &lex_table,
const map<rules::Symbol, string> &symbol_names) {
return CCodeGenerator(name, parse_table, lex_table, symbol_names).code();
const PreparedGrammar &syntax_grammar,
const PreparedGrammar &lexical_grammar) {
return CCodeGenerator(name, parse_table, lex_table, syntax_grammar, lexical_grammar).code();
}
}
}

5
src/compiler/generate_code/c_code.h
View file

@ -7,11 +7,14 @@
#include "compiler/lex_table.h"
namespace tree_sitter {
class PreparedGrammar;
namespace generate_code {
std::string c_code(std::string name,
const ParseTable &parse_table,
const LexTable &lex_table,
const std::map<rules::Symbol, std::string> &symbol_names);
const PreparedGrammar &syntax_grammar,
const PreparedGrammar &lexical_grammar);
}
}

25
src/compiler/generate_code/token_description.cc Normal file
View file

@ -0,0 +1,25 @@
#include "compiler/generate_code/token_description.h"
#include "tree_sitter/compiler.h"
#include "compiler/rules/visitor.h"
#include "compiler/rules/pattern.h"
#include "compiler/rules/string.h"
namespace tree_sitter {
using std::string;
namespace generate_code {
class TokenDescription : public rules::RuleFn<string> {
string apply_to(const rules::Pattern *rule) {
return "/" + rule->value + "/";
}
string apply_to(const rules::String *rule) {
return "'" + rule->value + "'";
}
};
std::string token_description(const rules::rule_ptr &rule) {
return TokenDescription().apply(rule);
}
}
}

13
src/compiler/generate_code/token_description.h Normal file
View file

@ -0,0 +1,13 @@
#ifndef COMPILER_GENERATE_CODE_TOKEN_DESCRIPTION_H_
#define COMPILER_GENERATE_CODE_TOKEN_DESCRIPTION_H_
#include <string>
#include "tree_sitter/compiler.h"
namespace tree_sitter {
namespace generate_code {
std::string token_description(const rules::rule_ptr &);
}
}
#endif // COMPILER_GENERATE_CODE_TOKEN_DESCRIPTION_H_

15
src/compiler/lex_table.cc
View file

@ -1,32 +1,33 @@
#include "compiler/lex_table.h"
#include "compiler/rules/interned_symbol.h"
namespace tree_sitter {
using std::string;
using std::to_string;
using std::map;
using std::set;
using rules::Symbol;
using rules::ISymbol;
using rules::CharacterSet;
LexAction::LexAction() :
type(LexActionTypeError),
symbol(Symbol("")),
symbol(ISymbol(-1)),
state_index(-1) {}
LexAction::LexAction(LexActionType type, size_t state_index, Symbol symbol) :
LexAction::LexAction(LexActionType type, size_t state_index, ISymbol symbol) :
type(type),
symbol(symbol),
state_index(state_index) {}
LexAction LexAction::Error() {
return LexAction(LexActionTypeError, -1, Symbol(""));
return LexAction(LexActionTypeError, -1, ISymbol(-1));
}
LexAction LexAction::Advance(size_t state_index) {
return LexAction(LexActionTypeAdvance, state_index, Symbol(""));
return LexAction(LexActionTypeAdvance, state_index, ISymbol(-1));
}
LexAction LexAction::Accept(Symbol symbol) {
LexAction LexAction::Accept(ISymbol symbol) {
return LexAction(LexActionTypeAccept, -1, symbol);
}
@ -42,7 +43,7 @@ namespace tree_sitter {
case LexActionTypeError:
return stream << string("#<error>");
case LexActionTypeAccept:
return stream << string("#<accept ") + action.symbol.name + ">";
return stream << string("#<accept ") + to_string(action.symbol.index) + ">";
case LexActionTypeAdvance:
return stream << string("#<advance ") + to_string(action.state_index) + ">";
default:

10
src/compiler/lex_table.h
View file

@ -5,7 +5,7 @@
#include <vector>
#include <set>
#include <string>
#include "compiler/rules/symbol.h"
#include "compiler/rules/interned_symbol.h"
#include "compiler/rules/character_set.h"
namespace tree_sitter {
@ -16,16 +16,16 @@ namespace tree_sitter {
} LexActionType;
class LexAction {
LexAction(LexActionType type, size_t state_index, rules::Symbol symbol);
LexAction(LexActionType type, size_t state_index, rules::ISymbol symbol);
public:
LexAction();
static LexAction Accept(rules::Symbol symbol);
static LexAction Accept(rules::ISymbol symbol);
static LexAction Error();
static LexAction Advance(size_t state_index);
bool operator==(const LexAction &action) const;
LexActionType type;
rules::Symbol symbol;
rules::ISymbol symbol;
size_t state_index;
};
@ -37,7 +37,7 @@ namespace std {
struct hash<tree_sitter::LexAction> {
size_t operator()(const tree_sitter::LexAction &action) const {
return (hash<int>()(action.type) ^
hash<tree_sitter::rules::Symbol>()(action.symbol) ^
hash<tree_sitter::rules::ISymbol>()(action.symbol) ^
hash<size_t>()(action.state_index));
}
};

52
src/compiler/name_symbols/name_symbols.cc
View file

@ -1,52 +0,0 @@
#include "compiler/name_symbols/name_symbols.h"
#include "compiler/prepared_grammar.h"
#include "compiler/rules/rule.h"
#include "compiler/rules/visitor.h"
#include "compiler/rules/pattern.h"
#include "compiler/rules/string.h"
#include "compiler/util/string_helpers.h"
#include "compiler/rules/built_in_symbols.h"
namespace tree_sitter {
namespace name_symbols {
using std::map;
using std::set;
using std::string;
using rules::Symbol;
using rules::SymbolTypeNormal;
using rules::SymbolTypeAuxiliary;
class TokenName : public rules::RuleFn<string> {
protected:
string apply_to(const rules::Pattern *rule) {
return "/" + util::escape_string(rule->value) + "/";
}
string apply_to(const rules::String *rule) {
return "'" + util::escape_string(rule->value) + "'";
}
};
map<rules::Symbol, string> name_symbols(const PreparedGrammar &syntactic_grammar,
const PreparedGrammar &lexical_grammar) {
map<rules::Symbol, string> result;
for (const auto &pair : syntactic_grammar.rules)
result.insert({ Symbol(pair.first, SymbolTypeNormal), pair.first });
for (const auto &pair : lexical_grammar.rules)
result.insert({ Symbol(pair.first, SymbolTypeNormal), pair.first });
for (const auto &pair : syntactic_grammar.aux_rules)
result.insert({ Symbol(pair.first, SymbolTypeAuxiliary), pair.first });
for (const auto &pair : lexical_grammar.aux_rules)
result.insert({
Symbol(pair.first, SymbolTypeAuxiliary),
TokenName().apply(pair.second)
});
result.insert({ rules::END_OF_INPUT(), "EOF" });
result.insert({ rules::ERROR(), "ERROR" });
return result;
}
}
}

18
src/compiler/name_symbols/name_symbols.h
View file

@ -1,18 +0,0 @@
#ifndef COMPILER_NAME_SYMBOLS_NAME_SYMBOLS_H_
#define COMPILER_NAME_SYMBOLS_NAME_SYMBOLS_H_
#include <map>
#include <set>
#include <string>
#include "compiler/rules/symbol.h"
namespace tree_sitter {
class PreparedGrammar;
namespace name_symbols {
std::map<rules::Symbol, std::string> name_symbols(const PreparedGrammar &syntactic_grammar,
const PreparedGrammar &lexical_grammar);
}
}
#endif // COMPILER_NAME_SYMBOLS_NAME_SYMBOLS_H_

22
src/compiler/parse_table.cc
View file

@ -7,11 +7,11 @@ namespace tree_sitter {
using std::to_string;
using std::set;
using std::vector;
using rules::Symbol;
using rules::ISymbol;
ParseAction::ParseAction(ParseActionType type,
size_t state_index,
Symbol symbol,
ISymbol symbol,
size_t consumed_symbol_count,
set<int> precedence_values) :
type(type),
@ -22,23 +22,23 @@ namespace tree_sitter {
ParseAction::ParseAction() :
type(ParseActionTypeError),
symbol(Symbol("")),
symbol(ISymbol(-1)),
state_index(-1),
consumed_symbol_count(0) {}
ParseAction ParseAction::Error() {
return ParseAction(ParseActionTypeError, -1, Symbol(""), 0, { 0 });
return ParseAction(ParseActionTypeError, -1, ISymbol(-1), 0, { 0 });
}
ParseAction ParseAction::Accept() {
return ParseAction(ParseActionTypeAccept, -1, Symbol(""), 0, { 0 });
return ParseAction(ParseActionTypeAccept, -1, ISymbol(-1), 0, { 0 });
}
ParseAction ParseAction::Shift(size_t state_index, set<int> precedence_values) {
return ParseAction(ParseActionTypeShift, state_index, Symbol(""), 0, precedence_values);
return ParseAction(ParseActionTypeShift, state_index, ISymbol(-1), 0, precedence_values);
}
ParseAction ParseAction::Reduce(Symbol symbol, size_t consumed_symbol_count, int precedence) {
ParseAction ParseAction::Reduce(ISymbol symbol, size_t consumed_symbol_count, int precedence) {
return ParseAction(ParseActionTypeReduce, -1, symbol, consumed_symbol_count, { precedence });
}
@ -58,7 +58,7 @@ namespace tree_sitter {
case ParseActionTypeShift:
return stream << (string("#<shift ") + to_string(action.state_index) + ">");
case ParseActionTypeReduce:
return stream << (string("#<reduce ") + action.symbol.name + ">");
return stream << (string("#<reduce ") + to_string(action.symbol.index) + ">");
default:
return stream;
}
@ -66,8 +66,8 @@ namespace tree_sitter {
ParseState::ParseState() : lex_state_id(-1) {}
set<Symbol> ParseState::expected_inputs() const {
set<Symbol> result;
set<ISymbol> ParseState::expected_inputs() const {
set<ISymbol> result;
for (auto &pair : actions)
result.insert(pair.first);
return result;
@ -90,7 +90,7 @@ namespace tree_sitter {
return states.size() - 1;
}
void ParseTable::add_action(ParseStateId id, Symbol symbol, ParseAction action) {
void ParseTable::add_action(ParseStateId id, ISymbol symbol, ParseAction action) {
symbols.insert(symbol);
states[id].actions[symbol] = action;
}

18
src/compiler/parse_table.h
View file

@ -6,7 +6,7 @@
#include <utility>
#include <vector>
#include "compiler/lex_table.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/interned_symbol.h"
namespace tree_sitter {
typedef enum {
@ -19,7 +19,7 @@ namespace tree_sitter {
class ParseAction {
ParseAction(ParseActionType type,
size_t state_index,
rules::Symbol symbol,
rules::ISymbol symbol,
size_t consumed_symbol_count,
std::set<int> precedence_values);
public:
@ -27,11 +27,11 @@ namespace tree_sitter {
static ParseAction Accept();
static ParseAction Error();
static ParseAction Shift(size_t state_index, std::set<int> precedence_values);
static ParseAction Reduce(rules::Symbol symbol, size_t consumed_symbol_count, int precedence);
static ParseAction Reduce(rules::ISymbol symbol, size_t consumed_symbol_count, int precedence);
bool operator==(const ParseAction &action) const;
ParseActionType type;
rules::Symbol symbol;
rules::ISymbol symbol;
size_t state_index;
size_t consumed_symbol_count;
std::set<int> precedence_values;
@ -46,7 +46,7 @@ namespace std {
size_t operator()(const tree_sitter::ParseAction &action) const {
return (
hash<int>()(action.type) ^
hash<tree_sitter::rules::Symbol>()(action.symbol) ^
hash<tree_sitter::rules::ISymbol>()(action.symbol) ^
hash<size_t>()(action.state_index) ^
hash<size_t>()(action.consumed_symbol_count));
}
@ -57,8 +57,8 @@ namespace tree_sitter {
class ParseState {
public:
ParseState();
std::map<rules::Symbol, ParseAction> actions;
std::set<rules::Symbol> expected_inputs() const;
std::map<rules::ISymbol, ParseAction> actions;
std::set<rules::ISymbol> expected_inputs() const;
LexStateId lex_state_id;
};
@ -69,10 +69,10 @@ namespace tree_sitter {
class ParseTable {
public:
uint64_t add_state();
void add_action(ParseStateId state_id, rules::Symbol symbol, ParseAction action);
void add_action(ParseStateId state_id, rules::ISymbol symbol, ParseAction action);
std::vector<ParseState> states;
std::set<rules::Symbol> symbols;
std::set<rules::ISymbol> symbols;
};
}

12
src/compiler/prepare_grammar/expand_repeats.cc
View file

@ -22,7 +22,7 @@ namespace tree_sitter {
using rules::Repeat;
using rules::Rule;
using rules::Seq;
using rules::Symbol;
using rules::ISymbol;
namespace prepare_grammar {
class ExpandRepeats : public rules::IdentityRuleFn {
@ -30,8 +30,9 @@ namespace tree_sitter {
rule_ptr apply_to(const Repeat *rule) {
rule_ptr inner_rule = apply(rule->content);
string helper_rule_name = rule_name + string("_repeat") + to_string(aux_rules.size() + 1);
rule_ptr repeat_symbol = make_shared<Symbol>(helper_rule_name, rules::SymbolTypeAuxiliary);
size_t index = aux_rules.size();
string helper_rule_name = rule_name + string("_repeat") + to_string(index);
rule_ptr repeat_symbol = make_shared<ISymbol>(offset + index, rules::SymbolOptionAuxiliary);
aux_rules.push_back({
helper_rule_name,
Choice::Build({
@ -43,8 +44,9 @@ namespace tree_sitter {
}
public:
ExpandRepeats(string rule_name) : rule_name(rule_name) {}
ExpandRepeats(string rule_name, size_t offset) : rule_name(rule_name), offset(offset) {}
size_t offset;
vector<pair<string, rules::rule_ptr>> aux_rules;
};
@ -52,7 +54,7 @@ namespace tree_sitter {
vector<pair<string, rules::rule_ptr>> rules, aux_rules(grammar.aux_rules);
for (auto &pair : grammar.rules) {
ExpandRepeats expander(pair.first);
ExpandRepeats expander(pair.first, aux_rules.size());
rules.push_back({ pair.first, expander.apply(pair.second) });
aux_rules.insert(aux_rules.end(), expander.aux_rules.begin(), expander.aux_rules.end());
}

104
src/compiler/prepare_grammar/extract_tokens.cc
View file

@ -1,4 +1,5 @@
#include "compiler/prepare_grammar/extract_tokens.h"
#include <map>
#include <vector>
#include <string>
#include "tree_sitter/compiler.h"
@ -15,69 +16,112 @@
namespace tree_sitter {
using std::pair;
using std::string;
using std::map;
using std::to_string;
using std::vector;
using std::make_shared;
using rules::rule_ptr;
using rules::ISymbol;
using std::dynamic_pointer_cast;
namespace prepare_grammar {
class IsToken : public rules::RuleFn<bool> {
bool apply_to(const rules::String *rule) {
return true;
bool apply_to(const rules::String *rule) { return true; }
bool apply_to(const rules::Pattern *rule) { return true; }
};
class SymbolInliner : public rules::IdentityRuleFn {
map<ISymbol, ISymbol> replacements;
using rules::IdentityRuleFn::apply_to;
int new_index_for_symbol(const ISymbol &symbol) {
int result = symbol.index;
for (const auto &pair : replacements)
if (pair.first.index < symbol.index &&
pair.first.is_auxiliary() == symbol.is_auxiliary())
result--;
return result;
}
bool apply_to(const rules::Pattern *rule) {
return true;
rule_ptr apply_to(const ISymbol *rule) {
auto replacement_pair = replacements.find(*rule);
if (replacement_pair != replacements.end())
return replacement_pair->second.copy();
else if (rule->is_built_in())
return rule->copy();
else
return make_shared<ISymbol>(new_index_for_symbol(*rule), rule->options);
}
public:
SymbolInliner(const map<ISymbol, ISymbol> &replacements, size_t rule_count, size_t aux_rule_count) :
replacements(replacements)
{}
};
class TokenExtractor : public rules::IdentityRuleFn {
string add_token(rule_ptr rule) {
for (auto pair : tokens)
if (*pair.second == *rule)
return pair.first;
string name = "token" + to_string(tokens.size() + 1);
tokens.push_back({ name, rule });
return name;
size_t add_token(rule_ptr rule) {
for (size_t i = 0; i < tokens.size(); i++)
if (tokens[i].second->operator==(*rule))
return i;
size_t index = tokens.size();
tokens.push_back({ "token" + to_string(index), rule });
return index;
}
rule_ptr default_apply(const rules::Rule *rule) {
auto result = rule->copy();
if (IsToken().apply(result)) {
return make_shared<rules::Symbol>(add_token(result), rules::SymbolTypeAuxiliary);
size_t index = add_token(result);
return make_shared<rules::ISymbol>(index, rules::SymbolOption(rules::SymbolOptionToken|rules::SymbolOptionAuxiliary));
} else {
return result;
}
}
public:
vector<pair<string, rules::rule_ptr>> tokens;
vector<pair<string, rule_ptr>> tokens;
};
pair<PreparedGrammar, PreparedGrammar> extract_tokens(const PreparedGrammar &input_grammar) {
vector<pair<string, rules::rule_ptr>> rules, tokens, aux_rules, aux_tokens;
vector<pair<string, rule_ptr>> rules, tokens, aux_rules, aux_tokens;
TokenExtractor extractor;
map<ISymbol, ISymbol> symbol_replacements;
for (auto &pair : input_grammar.rules) {
string name = pair.first;
rule_ptr rule = pair.second;
if (IsToken().apply(rule))
tokens.push_back({ name, rule });
else
rules.push_back({ name, extractor.apply(rule) });
for (size_t i = 0; i < input_grammar.rules.size(); i++) {
auto pair = input_grammar.rules[i];
if (IsToken().apply(pair.second)) {
tokens.push_back(pair);
symbol_replacements.insert({
ISymbol(i),
ISymbol(tokens.size() - 1, rules::SymbolOption(rules::SymbolOptionToken))
});
} else {
rules.push_back({ pair.first, extractor.apply(pair.second) });
}
}
for (auto &pair : input_grammar.aux_rules) {
string name = pair.first;
rule_ptr rule = pair.second;
if (IsToken().apply(rule))
aux_tokens.push_back({ name, rule });
else
aux_rules.push_back({ name, extractor.apply(rule) });
for (size_t i = 0; i < input_grammar.aux_rules.size(); i++) {
auto pair = input_grammar.aux_rules[i];
if (IsToken().apply(pair.second)) {
aux_tokens.push_back(pair);
symbol_replacements.insert({
ISymbol(i, rules::SymbolOptionAuxiliary),
ISymbol(aux_tokens.size() - 1, rules::SymbolOption(rules::SymbolOptionAuxiliary|rules::SymbolOptionToken))
});
} else {
aux_rules.push_back({ pair.first, extractor.apply(pair.second) });
}
}
aux_tokens.insert(aux_tokens.end(), extractor.tokens.begin(), extractor.tokens.end());
SymbolInliner inliner(symbol_replacements, input_grammar.rules.size(), input_grammar.aux_rules.size());
for (auto &pair : rules)
pair.second = inliner.apply(pair.second);
for (auto &pair : aux_rules)
pair.second = inliner.apply(pair.second);
return {
PreparedGrammar(rules, aux_rules),
PreparedGrammar(tokens, aux_tokens)

65
src/compiler/prepare_grammar/intern_symbols.cc Normal file
View file

@ -0,0 +1,65 @@
#include "compiler/prepare_grammar/intern_symbols.h"
#include <memory>
#include "tree_sitter/compiler.h"
#include "compiler/prepared_grammar.h"
#include "compiler/rules/visitor.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/interned_symbol.h"
namespace tree_sitter {
using std::string;
using rules::rule_ptr;
using std::vector;
using std::pair;
using std::make_shared;
using std::exception;
GrammarError::GrammarError(string rule_name) : rule_name(rule_name) {}
string GrammarError::message() const {
return "Undefined rule '" + rule_name + "'";
}
namespace prepare_grammar {
class InternSymbols : public rules::IdentityRuleFn {
const Grammar grammar;
using rules::IdentityRuleFn::apply_to;
long index_of(string rule_name) {
for (size_t i = 0; i < grammar.rules.size(); i++)
if (grammar.rules[i].first == rule_name)
return i;
return -1;
}
rule_ptr apply_to(const rules::Symbol *rule) {
long index = index_of(rule->name);
if (index == -1)
missing_rule_name = rule->name;
return make_shared<rules::ISymbol>(index);
}
public:
InternSymbols(const Grammar &grammar) : grammar(grammar) {}
string missing_rule_name;
};
pair<PreparedGrammar, const GrammarError *> intern_symbols(const Grammar &grammar) {
InternSymbols interner(grammar);
vector<pair<string, rule_ptr>> rules;
for (auto &pair : grammar.rules) {
auto new_rule = interner.apply(pair.second);
if (!interner.missing_rule_name.empty())
return {
PreparedGrammar(rules),
new GrammarError(interner.missing_rule_name)
};
rules.push_back({ pair.first, new_rule });
}
return { PreparedGrammar(rules), nullptr };
}
}
}

23
src/compiler/prepare_grammar/intern_symbols.h Normal file
View file

@ -0,0 +1,23 @@
#ifndef COMPILER_PREPARE_GRAMMAR_INTERN_SYMBOLS_H_
#define COMPILER_PREPARE_GRAMMAR_INTERN_SYMBOLS_H_
#include <utility>
#include <string>
namespace tree_sitter {
class Grammar;
class PreparedGrammar;
class GrammarError {
std::string rule_name;
public:
GrammarError(std::string rule_name);
std::string message() const;
};
namespace prepare_grammar {
std::pair<PreparedGrammar, const GrammarError *> intern_symbols(const Grammar &);
}
}
#endif // COMPILER_PREPARE_GRAMMAR_INTERN_SYMBOLS_H_

12
src/compiler/prepare_grammar/prepare_grammar.cc
View file

@ -2,15 +2,21 @@
#include "compiler/prepared_grammar.h"
#include "compiler/prepare_grammar/extract_tokens.h"
#include "compiler/prepare_grammar/expand_repeats.h"
#include "compiler/prepare_grammar/intern_symbols.h"
namespace tree_sitter {
using std::pair;
namespace prepare_grammar {
pair<PreparedGrammar, PreparedGrammar> prepare_grammar(const Grammar &input_grammar) {
auto grammars = prepare_grammar::extract_tokens(input_grammar);
auto rule_grammar = expand_repeats(grammars.first);
auto lex_grammar = grammars.second;
auto interned = intern_symbols(input_grammar);
if (interned.second) {
printf("Error!");
exit(1);
}
auto grammars = extract_tokens(interned.first);
const auto &rule_grammar = expand_repeats(grammars.first);
const auto &lex_grammar = grammars.second;
return { rule_grammar, lex_grammar };
}
}

33
src/compiler/prepared_grammar.cc
View file

@ -2,14 +2,14 @@
#include <vector>
#include <string>
#include <utility>
#include "compiler/rules/symbol.h"
#include "compiler/rules/interned_symbol.h"
namespace tree_sitter {
using std::string;
using std::pair;
using std::ostream;
using rules::rule_ptr;
using rules::Symbol;
using rules::ISymbol;
PreparedGrammar::PreparedGrammar(const std::vector<std::pair<std::string, rules::rule_ptr>> &rules,
const std::vector<std::pair<std::string, rules::rule_ptr>> &aux_rules) :
@ -20,12 +20,16 @@ namespace tree_sitter {
Grammar(grammar),
aux_rules({}) {}
const rule_ptr PreparedGrammar::rule(const Symbol &symbol) const {
auto rule_set = symbol.is_auxiliary() ? aux_rules : rules;
for (auto &pair : rule_set)
if (pair.first == symbol.name)
return pair.second;
return rule_ptr();
const rule_ptr PreparedGrammar::rule(const ISymbol &symbol) const {
return symbol.is_auxiliary() ?
aux_rules[symbol.index].second :
rules[symbol.index].second;
}
string PreparedGrammar::rule_name(const ISymbol &symbol) const {
return symbol.is_auxiliary() ?
aux_rules[symbol.index].first :
rules[symbol.index].first;
}
bool PreparedGrammar::operator==(const PreparedGrammar &other) const {
@ -40,19 +44,6 @@ namespace tree_sitter {
return true;
}
bool PreparedGrammar::has_definition(const Symbol &symbol) const {
return rule(symbol).get() != nullptr;
}
size_t PreparedGrammar::index_of(const rules::Symbol &symbol) const {
for (size_t i = 0; i < rules.size(); i++) {
if (rules[i].first == symbol.name) {
return i;
}
}
return -1;
}
ostream& operator<<(ostream &stream, const PreparedGrammar &grammar) {
stream << string("#<grammar");

7
src/compiler/prepared_grammar.h
View file

@ -5,7 +5,7 @@
#include <string>
#include <utility>
#include "tree_sitter/compiler.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/interned_symbol.h"
namespace tree_sitter {
class PreparedGrammar : public Grammar {
@ -15,9 +15,8 @@ namespace tree_sitter {
PreparedGrammar(const Grammar &grammar);
bool operator==(const PreparedGrammar &other) const;
bool has_definition(const rules::Symbol &symbol) const;
const rules::rule_ptr rule(const rules::Symbol &symbol) const;
size_t index_of(const rules::Symbol &symbol) const;
std::string rule_name(const rules::ISymbol &symbol) const;
const rules::rule_ptr rule(const rules::ISymbol &symbol) const;
const std::vector<std::pair<std::string, rules::rule_ptr>> aux_rules;
};

6
src/compiler/rules/built_in_symbols.cc
View file

@ -2,8 +2,8 @@
namespace tree_sitter {
namespace rules {
Symbol ERROR() { return Symbol("error", SymbolTypeBuiltIn); }
Symbol START() { return Symbol("start", SymbolTypeBuiltIn); }
Symbol END_OF_INPUT() { return Symbol("end", SymbolTypeBuiltIn); }
ISymbol END_OF_INPUT() { return ISymbol(-1, SymbolOptionToken); }
ISymbol ERROR() { return ISymbol(-2, SymbolOptionToken); }
ISymbol START() { return ISymbol(-3); }
}
}

8
src/compiler/rules/built_in_symbols.h
View file

@ -1,13 +1,13 @@
#ifndef COMPILER_RULES_BUILT_IN_SYMBOLS_H_
#define COMPILER_RULES_BUILT_IN_SYMBOLS_H_
#include "compiler/rules/symbol.h"
#include "compiler/rules/interned_symbol.h"
namespace tree_sitter {
namespace rules {
Symbol ERROR();
Symbol START();
Symbol END_OF_INPUT();
ISymbol ERROR();
ISymbol START();
ISymbol END_OF_INPUT();
}
}

61
src/compiler/rules/interned_symbol.cc Normal file
View file

@ -0,0 +1,61 @@
#include "compiler/rules/interned_symbol.h"
#include <map>
#include <string>
#include "compiler/rules/visitor.h"
namespace tree_sitter {
using std::string;
using std::to_string;
using std::hash;
namespace rules {
ISymbol::ISymbol(int index) :
index(index),
options(SymbolOption(0)) {}
ISymbol::ISymbol(int index, SymbolOption options) :
index(index),
options(options) {}
bool ISymbol::operator==(const Rule &rule) const {
const ISymbol *other = dynamic_cast<const ISymbol *>(&rule);
return other && (other->index == index) && (other->options == options);
}
size_t ISymbol::hash_code() const {
return hash<size_t>()(index) ^ hash<int16_t>()(options);
}
rule_ptr ISymbol::copy() const {
return std::make_shared<ISymbol>(*this);
}
string ISymbol::to_string() const {
string name = (options & SymbolOptionAuxiliary) ? "aux_" : "";
name += (options & SymbolOptionToken) ? "token" : "sym";
return "#<" + name + std::to_string(index) + ">";
}
bool ISymbol::operator<(const ISymbol &other) const {
if (options < other.options) return true;
if (options > other.options) return false;
return (index < other.index);
}
bool ISymbol::is_token() const {
return options & SymbolOptionToken;
}
bool ISymbol::is_built_in() const {
return index < 0;
}
bool ISymbol::is_auxiliary() const {
return options & SymbolOptionAuxiliary;
}
void ISymbol::accept(Visitor *visitor) const {
visitor->visit(this);
}
}
}

45
src/compiler/rules/interned_symbol.h Normal file
View file

@ -0,0 +1,45 @@
#ifndef COMPILER_RULES_INTERNED_SYMBOL_H_
#define COMPILER_RULES_INTERNED_SYMBOL_H_
#include "compiler/rules/symbol.h"
namespace tree_sitter {
namespace rules {
typedef enum {
SymbolOptionToken = 1 << 0,
SymbolOptionAuxiliary = 1 << 1,
} SymbolOption;
class ISymbol : public Rule {
public:
explicit ISymbol(int index);
ISymbol(int index, SymbolOption options);
bool operator==(const Rule& other) const;
size_t hash_code() const;
rule_ptr copy() const;
std::string to_string() const;
void accept(Visitor *visitor) const;
bool operator<(const ISymbol &other) const;
bool is_token() const;
bool is_built_in() const;
bool is_auxiliary() const;
int index;
SymbolOption options;
};
}
}
namespace std {
template<>
struct hash<tree_sitter::rules::ISymbol> {
size_t operator()(const tree_sitter::rules::ISymbol &rule) const {
return rule.hash_code();
}
};
}
#endif // COMPILER_RULES_INTERNED_SYMBOL_H_

34
src/compiler/rules/symbol.cc
View file

@ -8,8 +8,7 @@ namespace tree_sitter {
using std::hash;
namespace rules {
Symbol::Symbol(const std::string &name) : name(name), type(SymbolTypeNormal) {}
Symbol::Symbol(const std::string &name, SymbolType type) : name(name), type(type) {}
Symbol::Symbol(const std::string &name) : name(name) {}
bool Symbol::operator==(const Rule &rule) const {
const Symbol *other = dynamic_cast<const Symbol *>(&rule);
@ -17,11 +16,11 @@ namespace tree_sitter {
}
bool Symbol::operator==(const Symbol &other) const {
return (other.name == name) && (other.type == type);
return other.name == name;
}
size_t Symbol::hash_code() const {
return hash<string>()(name) ^ hash<int16_t>()(type);
return hash<string>()(name);
}
rule_ptr Symbol::copy() const {
@ -29,34 +28,11 @@ namespace tree_sitter {
}
string Symbol::to_string() const {
switch (type) {
case SymbolTypeNormal:
return string("#<sym '") + name + "'>";
case SymbolTypeAuxiliary:
return string("#<aux_sym '") + name + "'>";
case SymbolTypeBuiltIn:
return string("#<builtin_sym '") + name + "'>";
default:
return "";
}
return string("#<sym '") + name + "'>";
}
bool Symbol::operator<(const Symbol &other) const {
if (type < other.type) return true;
if (type > other.type) return false;
return (name < other.name);
}
bool Symbol::is_built_in() const {
return type == SymbolTypeBuiltIn;
}
bool Symbol::is_auxiliary() const {
return type == SymbolTypeAuxiliary;
}
bool Symbol::is_hidden() const {
return (name.front() == '_' || type == SymbolTypeAuxiliary);
return name < other.name;
}
void Symbol::accept(Visitor *visitor) const {

12
src/compiler/rules/symbol.h
View file

@ -7,16 +7,9 @@
namespace tree_sitter {
namespace rules {
typedef enum {
SymbolTypeNormal,
SymbolTypeAuxiliary,
SymbolTypeBuiltIn
} SymbolType;
class Symbol : public Rule {
public:
explicit Symbol(const std::string &name);
Symbol(const std::string &name, SymbolType type);
bool operator==(const Rule& other) const;
bool operator==(const Symbol &other) const;
@ -27,12 +20,7 @@ namespace tree_sitter {
void accept(Visitor *visitor) const;
bool operator<(const Symbol &other) const;
bool is_built_in() const;
bool is_hidden() const;
bool is_auxiliary() const;
std::string name;
SymbolType type;
};
}
}

5
src/compiler/rules/visitor.h
View file

@ -12,6 +12,7 @@ namespace tree_sitter {
class Repeat;
class Seq;
class String;
class ISymbol;
class Pattern;
class Metadata;
@ -26,6 +27,7 @@ namespace tree_sitter {
virtual void visit(const Seq *rule) = 0;
virtual void visit(const String *rule) = 0;
virtual void visit(const Symbol *rule) = 0;
virtual void visit(const ISymbol *rule) = 0;
};
template<typename T>
@ -48,6 +50,7 @@ namespace tree_sitter {
virtual T apply_to(const Seq *rule) { return default_apply((const Rule *)rule); }
virtual T apply_to(const String *rule) { return default_apply((const Rule *)rule); }
virtual T apply_to(const Symbol *rule) { return default_apply((const Rule *)rule); }
virtual T apply_to(const ISymbol *rule) { return default_apply((const Rule *)rule); }
void visit(const Blank *rule) { value_ = apply_to(rule); }
void visit(const CharacterSet *rule) { value_ = apply_to(rule); }
@ -58,12 +61,14 @@ namespace tree_sitter {
void visit(const Seq *rule) { value_ = apply_to(rule); }
void visit(const String *rule) { value_ = apply_to(rule); }
void visit(const Symbol *rule) { value_ = apply_to(rule); }
void visit(const ISymbol *rule) { value_ = apply_to(rule); }
private:
T value_;
};
class IdentityRuleFn : public RuleFn<rule_ptr> {
protected:
virtual rule_ptr default_apply(const Rule *rule);
virtual rule_ptr apply_to(const Choice *rule);
virtual rule_ptr apply_to(const Metadata *rule);