traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix expansion of repeat rules into recursive rules

Browse source 
Previously, the way repeat rules were expanded, the auxiliary
rule always needed to be reduced, even if the repeating content
was empty. This caused problems in parse states where some items
contained the repeat rule and some did not. To make those cases
work, the repeat rule had to explicitly be marked as optional.
With this change, that is no longer necessary.
This commit is contained in:
Max Brunsfeld 2014-09-07 08:50:42 -07:00
parent 43ecac2a1d
commit ed11ef557a
10 changed files with 48641 additions and 39051 deletions
Download patch file Download diff file Expand all files Collapse all files

6799
examples/parsers/golang.c
View file

File diff suppressed because it is too large Load diff

80259
examples/parsers/javascript.c
View file

File diff suppressed because it is too large Load diff

392
examples/parsers/json.c
View file

@ -1,6 +1,6 @@
#include "tree_sitter/parser.h"
#define STATE_COUNT 60
#define STATE_COUNT 69
#define SYMBOL_COUNT 19
enum {
@ -378,51 +378,60 @@ static TSStateId ts_lex_states[STATE_COUNT] = {
[12] = 32,
[13] = 1,
[14] = 30,
[15] = 32,
[16] = 30,
[17] = 33,
[18] = 34,
[19] = 30,
[20] = 1,
[21] = 33,
[15] = 30,
[16] = 32,
[17] = 30,
[18] = 33,
[19] = 34,
[20] = 30,
[21] = 1,
[22] = 33,
[23] = 26,
[24] = 28,
[25] = 30,
[26] = 33,
[27] = 32,
[28] = 33,
[23] = 33,
[24] = 26,
[25] = 28,
[26] = 30,
[27] = 33,
[28] = 32,
[29] = 33,
[30] = 34,
[30] = 33,
[31] = 33,
[32] = 1,
[32] = 34,
[33] = 33,
[34] = 36,
[35] = 37,
[36] = 33,
[37] = 34,
[38] = 1,
[34] = 1,
[35] = 33,
[36] = 36,
[37] = 37,
[38] = 33,
[39] = 33,
[40] = 36,
[41] = 36,
[40] = 34,
[41] = 1,
[42] = 33,
[43] = 36,
[44] = 33,
[45] = 36,
[46] = 30,
[47] = 36,
[48] = 30,
[49] = 32,
[50] = 24,
[51] = 33,
[52] = 34,
[53] = 24,
[54] = 1,
[55] = 33,
[56] = 36,
[44] = 36,
[45] = 33,
[46] = 36,
[47] = 33,
[48] = 33,
[49] = 36,
[50] = 30,
[51] = 30,
[52] = 36,
[53] = 30,
[54] = 30,
[55] = 32,
[56] = 24,
[57] = 24,
[58] = 36,
[59] = 24,
[58] = 33,
[59] = 34,
[60] = 24,
[61] = 1,
[62] = 33,
[63] = 36,
[64] = 24,
[65] = 24,
[66] = 36,
[67] = 24,
[68] = 24,
};
#pragma GCC diagnostic push
@ -448,9 +457,9 @@ static const TSParseAction ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_value, 1),
},
[3] = {
[ts_builtin_sym_error] = SHIFT(51),
[ts_sym_string] = SHIFT(52),
[ts_aux_sym_4] = SHIFT(53),
[ts_builtin_sym_error] = SHIFT(58),
[ts_sym_string] = SHIFT(59),
[ts_aux_sym_4] = SHIFT(60),
},
[4] = {
[ts_sym_value] = SHIFT(5),
@ -467,18 +476,18 @@ static const TSParseAction ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[ts_aux_sym_6] = SHIFT(9),
},
[5] = {
[ts_aux_sym_array_repeat0] = SHIFT(49),
[ts_aux_sym_array_repeat0] = SHIFT(55),
[ts_aux_sym_3] = SHIFT(13),
[ts_aux_sym_6] = REDUCE(ts_aux_sym_array_repeat0, 0),
[ts_aux_sym_6] = SHIFT(56),
},
[6] = {
[ts_aux_sym_3] = REDUCE(ts_sym_value, 1),
[ts_aux_sym_6] = REDUCE(ts_sym_value, 1),
},
[7] = {
[ts_builtin_sym_error] = SHIFT(17),
[ts_sym_string] = SHIFT(18),
[ts_aux_sym_4] = SHIFT(19),
[ts_builtin_sym_error] = SHIFT(18),
[ts_sym_string] = SHIFT(19),
[ts_aux_sym_4] = SHIFT(20),
},
[8] = {
[ts_sym_value] = SHIFT(10),
@ -500,20 +509,20 @@ static const TSParseAction ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[10] = {
[ts_aux_sym_array_repeat0] = SHIFT(12),
[ts_aux_sym_3] = SHIFT(13),
[ts_aux_sym_6] = REDUCE(ts_aux_sym_array_repeat0, 0),
[ts_aux_sym_6] = SHIFT(14),
},
[11] = {
[ts_aux_sym_3] = REDUCE(ts_sym_array, 2),
[ts_aux_sym_6] = REDUCE(ts_sym_array, 2),
},
[12] = {
[ts_aux_sym_6] = SHIFT(16),
[ts_aux_sym_6] = SHIFT(17),
},
[13] = {
[ts_sym_value] = SHIFT(14),
[ts_sym_value] = SHIFT(15),
[ts_sym_object] = SHIFT(6),
[ts_sym_array] = SHIFT(6),
[ts_builtin_sym_error] = SHIFT(14),
[ts_builtin_sym_error] = SHIFT(15),
[ts_sym_string] = SHIFT(6),
[ts_sym_number] = SHIFT(6),
[ts_sym_null] = SHIFT(6),
@ -523,60 +532,64 @@ static const TSParseAction ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[ts_aux_sym_5] = SHIFT(8),
},
[14] = {
[ts_aux_sym_array_repeat0] = SHIFT(15),
[ts_aux_sym_3] = SHIFT(13),
[ts_aux_sym_6] = REDUCE(ts_aux_sym_array_repeat0, 0),
[ts_aux_sym_3] = REDUCE(ts_sym_array, 3),
[ts_aux_sym_6] = REDUCE(ts_sym_array, 3),
},
[15] = {
[ts_aux_sym_6] = REDUCE(ts_aux_sym_array_repeat0, 3),
[ts_aux_sym_array_repeat0] = SHIFT(16),
[ts_aux_sym_3] = SHIFT(13),
[ts_aux_sym_6] = REDUCE(ts_aux_sym_array_repeat0, 2),
},
[16] = {
[ts_aux_sym_6] = REDUCE(ts_aux_sym_array_repeat0, 3),
},
[17] = {
[ts_aux_sym_3] = REDUCE(ts_sym_array, 4),
[ts_aux_sym_6] = REDUCE(ts_sym_array, 4),
},
[17] = {
[ts_aux_sym_object_repeat0] = SHIFT(47),
[ts_aux_sym_3] = SHIFT(35),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 0),
},
[18] = {
[ts_aux_sym_2] = SHIFT(20),
[ts_aux_sym_object_repeat0] = SHIFT(52),
[ts_aux_sym_3] = SHIFT(37),
[ts_aux_sym_4] = SHIFT(53),
},
[19] = {
[ts_aux_sym_2] = SHIFT(21),
},
[20] = {
[ts_aux_sym_3] = REDUCE(ts_sym_object, 2),
[ts_aux_sym_6] = REDUCE(ts_sym_object, 2),
},
[20] = {
[ts_sym_value] = SHIFT(21),
[ts_sym_object] = SHIFT(22),
[ts_sym_array] = SHIFT(22),
[ts_sym_string] = SHIFT(22),
[ts_sym_number] = SHIFT(22),
[ts_sym_null] = SHIFT(22),
[ts_sym_true] = SHIFT(22),
[ts_sym_false] = SHIFT(22),
[ts_aux_sym_1] = SHIFT(23),
[ts_aux_sym_5] = SHIFT(24),
},
[21] = {
[ts_aux_sym_object_repeat0] = SHIFT(45),
[ts_aux_sym_3] = SHIFT(35),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 0),
[ts_sym_value] = SHIFT(22),
[ts_sym_object] = SHIFT(23),
[ts_sym_array] = SHIFT(23),
[ts_sym_string] = SHIFT(23),
[ts_sym_number] = SHIFT(23),
[ts_sym_null] = SHIFT(23),
[ts_sym_true] = SHIFT(23),
[ts_sym_false] = SHIFT(23),
[ts_aux_sym_1] = SHIFT(24),
[ts_aux_sym_5] = SHIFT(25),
},
[22] = {
[ts_aux_sym_object_repeat0] = SHIFT(49),
[ts_aux_sym_3] = SHIFT(37),
[ts_aux_sym_4] = SHIFT(50),
},
[23] = {
[ts_aux_sym_3] = REDUCE(ts_sym_value, 1),
[ts_aux_sym_4] = REDUCE(ts_sym_value, 1),
},
[23] = {
[ts_builtin_sym_error] = SHIFT(29),
[ts_sym_string] = SHIFT(30),
[ts_aux_sym_4] = SHIFT(31),
},
[24] = {
[ts_sym_value] = SHIFT(25),
[ts_builtin_sym_error] = SHIFT(31),
[ts_sym_string] = SHIFT(32),
[ts_aux_sym_4] = SHIFT(33),
},
[25] = {
[ts_sym_value] = SHIFT(26),
[ts_sym_object] = SHIFT(6),
[ts_sym_array] = SHIFT(6),
[ts_builtin_sym_error] = SHIFT(25),
[ts_builtin_sym_error] = SHIFT(26),
[ts_sym_string] = SHIFT(6),
[ts_sym_number] = SHIFT(6),
[ts_sym_null] = SHIFT(6),
@ -584,160 +597,189 @@ static const TSParseAction ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] = {
[ts_sym_false] = SHIFT(6),
[ts_aux_sym_1] = SHIFT(7),
[ts_aux_sym_5] = SHIFT(8),
[ts_aux_sym_6] = SHIFT(26),
},
[25] = {
[ts_aux_sym_array_repeat0] = SHIFT(27),
[ts_aux_sym_3] = SHIFT(13),
[ts_aux_sym_6] = REDUCE(ts_aux_sym_array_repeat0, 0),
[ts_aux_sym_6] = SHIFT(27),
},
[26] = {
[ts_aux_sym_array_repeat0] = SHIFT(28),
[ts_aux_sym_3] = SHIFT(13),
[ts_aux_sym_6] = SHIFT(29),
},
[27] = {
[ts_aux_sym_3] = REDUCE(ts_sym_array, 2),
[ts_aux_sym_4] = REDUCE(ts_sym_array, 2),
},
[27] = {
[ts_aux_sym_6] = SHIFT(28),
},
[28] = {
[ts_aux_sym_6] = SHIFT(30),
},
[29] = {
[ts_aux_sym_3] = REDUCE(ts_sym_array, 3),
[ts_aux_sym_4] = REDUCE(ts_sym_array, 3),
},
[30] = {
[ts_aux_sym_3] = REDUCE(ts_sym_array, 4),
[ts_aux_sym_4] = REDUCE(ts_sym_array, 4),
},
[29] = {
[ts_aux_sym_object_repeat0] = SHIFT(43),
[ts_aux_sym_3] = SHIFT(35),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 0),
},
[30] = {
[ts_aux_sym_2] = SHIFT(32),
},
[31] = {
[ts_aux_sym_object_repeat0] = SHIFT(46),
[ts_aux_sym_3] = SHIFT(37),
[ts_aux_sym_4] = SHIFT(47),
},
[32] = {
[ts_aux_sym_2] = SHIFT(34),
},
[33] = {
[ts_aux_sym_3] = REDUCE(ts_sym_object, 2),
[ts_aux_sym_4] = REDUCE(ts_sym_object, 2),
},
[32] = {
[ts_sym_value] = SHIFT(33),
[ts_sym_object] = SHIFT(22),
[ts_sym_array] = SHIFT(22),
[ts_sym_string] = SHIFT(22),
[ts_sym_number] = SHIFT(22),
[ts_sym_null] = SHIFT(22),
[ts_sym_true] = SHIFT(22),
[ts_sym_false] = SHIFT(22),
[ts_aux_sym_1] = SHIFT(23),
[ts_aux_sym_5] = SHIFT(24),
},
[33] = {
[ts_aux_sym_object_repeat0] = SHIFT(34),
[ts_aux_sym_3] = SHIFT(35),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 0),
},
[34] = {
[ts_aux_sym_4] = SHIFT(42),
[ts_sym_value] = SHIFT(35),
[ts_sym_object] = SHIFT(23),
[ts_sym_array] = SHIFT(23),
[ts_sym_string] = SHIFT(23),
[ts_sym_number] = SHIFT(23),
[ts_sym_null] = SHIFT(23),
[ts_sym_true] = SHIFT(23),
[ts_sym_false] = SHIFT(23),
[ts_aux_sym_1] = SHIFT(24),
[ts_aux_sym_5] = SHIFT(25),
},
[35] = {
[ts_builtin_sym_error] = SHIFT(36),
[ts_sym_string] = SHIFT(37),
[ts_aux_sym_object_repeat0] = SHIFT(36),
[ts_aux_sym_3] = SHIFT(37),
[ts_aux_sym_4] = SHIFT(38),
},
[36] = {
[ts_aux_sym_object_repeat0] = SHIFT(41),
[ts_aux_sym_3] = SHIFT(35),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 0),
[ts_aux_sym_4] = SHIFT(45),
},
[37] = {
[ts_aux_sym_2] = SHIFT(38),
[ts_builtin_sym_error] = SHIFT(39),
[ts_sym_string] = SHIFT(40),
},
[38] = {
[ts_sym_value] = SHIFT(39),
[ts_sym_object] = SHIFT(22),
[ts_sym_array] = SHIFT(22),
[ts_sym_string] = SHIFT(22),
[ts_sym_number] = SHIFT(22),
[ts_sym_null] = SHIFT(22),
[ts_sym_true] = SHIFT(22),
[ts_sym_false] = SHIFT(22),
[ts_aux_sym_1] = SHIFT(23),
[ts_aux_sym_5] = SHIFT(24),
[ts_aux_sym_3] = REDUCE(ts_sym_object, 5),
[ts_aux_sym_4] = REDUCE(ts_sym_object, 5),
},
[39] = {
[ts_aux_sym_object_repeat0] = SHIFT(40),
[ts_aux_sym_3] = SHIFT(35),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 0),
[ts_aux_sym_object_repeat0] = SHIFT(44),
[ts_aux_sym_3] = SHIFT(37),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 2),
},
[40] = {
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 5),
[ts_aux_sym_2] = SHIFT(41),
},
[41] = {
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 3),
[ts_sym_value] = SHIFT(42),
[ts_sym_object] = SHIFT(23),
[ts_sym_array] = SHIFT(23),
[ts_sym_string] = SHIFT(23),
[ts_sym_number] = SHIFT(23),
[ts_sym_null] = SHIFT(23),
[ts_sym_true] = SHIFT(23),
[ts_sym_false] = SHIFT(23),
[ts_aux_sym_1] = SHIFT(24),
[ts_aux_sym_5] = SHIFT(25),
},
[42] = {
[ts_aux_sym_object_repeat0] = SHIFT(43),
[ts_aux_sym_3] = SHIFT(37),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 4),
},
[43] = {
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 5),
},
[44] = {
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 3),
},
[45] = {
[ts_aux_sym_3] = REDUCE(ts_sym_object, 6),
[ts_aux_sym_4] = REDUCE(ts_sym_object, 6),
},
[43] = {
[ts_aux_sym_4] = SHIFT(44),
},
[44] = {
[ts_aux_sym_3] = REDUCE(ts_sym_object, 4),
[ts_aux_sym_4] = REDUCE(ts_sym_object, 4),
},
[45] = {
[ts_aux_sym_4] = SHIFT(46),
},
[46] = {
[ts_aux_sym_3] = REDUCE(ts_sym_object, 6),
[ts_aux_sym_6] = REDUCE(ts_sym_object, 6),
[ts_aux_sym_4] = SHIFT(48),
},
[47] = {
[ts_aux_sym_4] = SHIFT(48),
[ts_aux_sym_3] = REDUCE(ts_sym_object, 3),
[ts_aux_sym_4] = REDUCE(ts_sym_object, 3),
},
[48] = {
[ts_aux_sym_3] = REDUCE(ts_sym_object, 4),
[ts_aux_sym_6] = REDUCE(ts_sym_object, 4),
[ts_aux_sym_4] = REDUCE(ts_sym_object, 4),
},
[49] = {
[ts_aux_sym_6] = SHIFT(50),
[ts_aux_sym_4] = SHIFT(51),
},
[50] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_array, 4),
[ts_aux_sym_3] = REDUCE(ts_sym_object, 5),
[ts_aux_sym_6] = REDUCE(ts_sym_object, 5),
},
[51] = {
[ts_aux_sym_object_repeat0] = SHIFT(58),
[ts_aux_sym_3] = SHIFT(35),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 0),
[ts_aux_sym_3] = REDUCE(ts_sym_object, 6),
[ts_aux_sym_6] = REDUCE(ts_sym_object, 6),
},
[52] = {
[ts_aux_sym_2] = SHIFT(54),
[ts_aux_sym_4] = SHIFT(54),
},
[53] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_object, 2),
[ts_aux_sym_3] = REDUCE(ts_sym_object, 3),
[ts_aux_sym_6] = REDUCE(ts_sym_object, 3),
},
[54] = {
[ts_sym_value] = SHIFT(55),
[ts_sym_object] = SHIFT(22),
[ts_sym_array] = SHIFT(22),
[ts_sym_string] = SHIFT(22),
[ts_sym_number] = SHIFT(22),
[ts_sym_null] = SHIFT(22),
[ts_sym_true] = SHIFT(22),
[ts_sym_false] = SHIFT(22),
[ts_aux_sym_1] = SHIFT(23),
[ts_aux_sym_5] = SHIFT(24),
[ts_aux_sym_3] = REDUCE(ts_sym_object, 4),
[ts_aux_sym_6] = REDUCE(ts_sym_object, 4),
},
[55] = {
[ts_aux_sym_object_repeat0] = SHIFT(56),
[ts_aux_sym_3] = SHIFT(35),
[ts_aux_sym_4] = REDUCE(ts_aux_sym_object_repeat0, 0),
[ts_aux_sym_6] = SHIFT(57),
},
[56] = {
[ts_aux_sym_4] = SHIFT(57),
[ts_builtin_sym_end] = REDUCE(ts_sym_array, 3),
},
[57] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_object, 6),
[ts_builtin_sym_end] = REDUCE(ts_sym_array, 4),
},
[58] = {
[ts_aux_sym_4] = SHIFT(59),
[ts_aux_sym_object_repeat0] = SHIFT(66),
[ts_aux_sym_3] = SHIFT(37),
[ts_aux_sym_4] = SHIFT(67),
},
[59] = {
[ts_aux_sym_2] = SHIFT(61),
},
[60] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_object, 2),
},
[61] = {
[ts_sym_value] = SHIFT(62),
[ts_sym_object] = SHIFT(23),
[ts_sym_array] = SHIFT(23),
[ts_sym_string] = SHIFT(23),
[ts_sym_number] = SHIFT(23),
[ts_sym_null] = SHIFT(23),
[ts_sym_true] = SHIFT(23),
[ts_sym_false] = SHIFT(23),
[ts_aux_sym_1] = SHIFT(24),
[ts_aux_sym_5] = SHIFT(25),
},
[62] = {
[ts_aux_sym_object_repeat0] = SHIFT(63),
[ts_aux_sym_3] = SHIFT(37),
[ts_aux_sym_4] = SHIFT(64),
},
[63] = {
[ts_aux_sym_4] = SHIFT(65),
},
[64] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_object, 5),
},
[65] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_object, 6),
},
[66] = {
[ts_aux_sym_4] = SHIFT(68),
},
[67] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_object, 3),
},
[68] = {
[ts_builtin_sym_end] = REDUCE(ts_sym_object, 4),
},
};

70
spec/compiler/prepare_grammar/expand_repeats_spec.cc
View file

@ -8,7 +8,7 @@ START_TEST
using namespace rules;
using prepare_grammar::expand_repeats;
describe("expanding repeat rules in a grammar", []() {
describe("expand_repeats", []() {
it("replaces repeat rules with pairs of recursive rules", [&]() {
SyntaxGrammar grammar({
{ "rule0", repeat(i_token(0)) },
@ -17,29 +17,35 @@ describe("expanding repeat rules in a grammar", []() {
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", i_aux_sym(0) },
{ "rule0", choice({ i_aux_sym(0), blank() }) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({ seq({ i_token(0), i_aux_sym(0) }), blank() }) },
{ "rule0_repeat0", seq({
i_token(0),
choice({ i_aux_sym(0), blank() }) }) },
})));
});
it("replaces repeats inside of sequences", [&]() {
SyntaxGrammar grammar({
{ "rule0", seq({ i_token(10), repeat(i_token(11)) }) },
{ "rule0", seq({
i_token(10),
repeat(i_token(11)) }) },
}, {}, set<Symbol>());
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", seq({ i_token(10), i_aux_sym(0) }) },
{ "rule0", seq({
i_token(10),
choice({ i_aux_sym(0), blank() }) }) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({ i_token(11), i_aux_sym(0) }),
blank() }) },
{ "rule0_repeat0", seq({
i_token(11),
choice({ i_aux_sym(0), blank() }) }) },
})));
});
@ -51,38 +57,38 @@ describe("expanding repeat rules in a grammar", []() {
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", choice({ i_token(10), i_aux_sym(0) }) },
{ "rule0", choice({ i_token(10), i_aux_sym(0), blank() }) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({ i_token(11), i_aux_sym(0) }),
blank() }) },
{ "rule0_repeat0", seq({
i_token(11),
choice({ i_aux_sym(0), blank() }) }) },
})));
});
it("can replace multiple repeats in the same rule", [&]() {
SyntaxGrammar grammar({
{ "rule0", seq({ repeat(i_token(10)), repeat(i_token(11)) }) },
{ "rule0", seq({
repeat(i_token(10)),
repeat(i_token(11)) }) },
}, {}, set<Symbol>());
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", seq({ i_aux_sym(0), i_aux_sym(1) }) },
{ "rule0", seq({
choice({ i_aux_sym(0), blank() }),
choice({ i_aux_sym(1), blank() }) }) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({
i_token(10),
i_aux_sym(0) }),
blank() }) },
{ "rule0_repeat1", choice({
seq({
i_token(11),
i_aux_sym(1) }),
blank() }) },
{ "rule0_repeat0", seq({
i_token(10),
choice({ i_aux_sym(0), blank() }) }) },
{ "rule0_repeat1", seq({
i_token(11),
choice({ i_aux_sym(1), blank() }) }) },
})));
});
@ -95,17 +101,17 @@ describe("expanding repeat rules in a grammar", []() {
auto match = expand_repeats(grammar);
AssertThat(match.rules, Equals(rule_list({
{ "rule0", i_aux_sym(0) },
{ "rule1", i_aux_sym(1) },
{ "rule0", choice({ i_aux_sym(0), blank() }) },
{ "rule1", choice({ i_aux_sym(1), blank() }) },
})));
AssertThat(match.aux_rules, Equals(rule_list({
{ "rule0_repeat0", choice({
seq({ i_token(10), i_aux_sym(0) }),
blank() }) },
{ "rule1_repeat0", choice({
seq({ i_token(11), i_aux_sym(1) }),
blank() }) },
{ "rule0_repeat0", seq({
i_token(10),
choice({ i_aux_sym(0), blank() }) }) },
{ "rule1_repeat0", seq({
i_token(11),
choice({ i_aux_sym(1), blank() }) }) },
})));
});
});

10
spec/runtime/document_spec.cc
View file

@ -326,7 +326,7 @@ describe("Document", [&]() {
it("is incorporated into the tree", [&]() {
ts_document_set_input_string(doc, "fn()\n");
AssertThat(ts_node_string(ts_document_root_node(doc)), Equals(
"(DOCUMENT (program (expression_statement (function_call (identifier)))))"));
"(DOCUMENT (expression_statement (function_call (identifier))))"));
});
});
@ -336,9 +336,9 @@ describe("Document", [&]() {
"fn()\n"
" .otherFn();");
AssertThat(ts_node_string(ts_document_root_node(doc)), Equals(
"(DOCUMENT (program "
"(DOCUMENT "
"(expression_statement (function_call "
"(property_access (function_call (identifier)) (identifier))))))"));
"(property_access (function_call (identifier)) (identifier)))))"));
});
describe("when several ubiquitous tokens appear in a row", [&]() {
@ -349,11 +349,11 @@ describe("Document", [&]() {
"\n\n"
".otherFn();");
AssertThat(ts_node_string(ts_document_root_node(doc)), Equals(
"(DOCUMENT (program "
"(DOCUMENT "
"(expression_statement (function_call "
"(property_access (function_call (identifier)) "
"(comment) "
"(identifier))))))"));
"(identifier)))))"));
});
});
});

53
spec/runtime/languages/javascript/control_flow.txt
View file

@ -20,9 +20,8 @@ if (isReady()) {
console.log(theData)
}
---
(program
(if_statement (function_call (identifier))
(statement_block (expression_statement (function_call (property_access (identifier) (identifier)) (identifier))))))
(if_statement (function_call (identifier))
(statement_block (expression_statement (function_call (property_access (identifier) (identifier)) (identifier)))))
==========================================
parses if-else statements
@ -33,10 +32,10 @@ if (theCondition) {
secondFunction();
}
---
(program (if_statement
(if_statement
(identifier)
(statement_block (expression_statement (function_call (identifier))))
(statement_block (expression_statement (function_call (identifier))))))
(statement_block (expression_statement (function_call (identifier)))))
==================================================
parses if-else statements with multiple conditions
@ -49,12 +48,11 @@ else {
thirdFunction();
}
---
(program
(if_statement (identifier)
(statement_block (expression_statement (function_call (identifier))))
(if_statement (identifier)
(statement_block (expression_statement (function_call (identifier))))
(if_statement (identifier)
(expression_statement (function_call (identifier)))
(statement_block (expression_statement (function_call (identifier)))))))
(expression_statement (function_call (identifier)))
(statement_block (expression_statement (function_call (identifier))))))
==========================================
parses for loops
@ -63,11 +61,11 @@ for (var i = 1; someCondition(i); i = next()) {
doSomething();
}
---
(program (for_statement
(for_statement
(var_declaration (identifier) (number))
(expression_statement (function_call (identifier) (identifier)))
(assignment (identifier) (function_call (identifier)))
(statement_block (expression_statement (function_call (identifier))))))
(statement_block (expression_statement (function_call (identifier)))))
==========================================
parses for-in loops
@ -92,9 +90,9 @@ while (someCondition(i)) {
doSomething();
}
---
(program (while_statement
(while_statement
(function_call (identifier) (identifier))
(statement_block (expression_statement (function_call (identifier))))))
(statement_block (expression_statement (function_call (identifier)))))
==========================================
parses try/catch statements
@ -126,8 +124,7 @@ parses throw statements
===========================================
throw new Error("wtf");
---
(program
(throw_statement (constructor_call (function_call (identifier) (string)))))
(throw_statement (constructor_call (function_call (identifier) (string))))
===========================================
parses indented code after blocks
@ -154,16 +151,14 @@ switch(x) {
print("three");
}
---
(program
(switch_statement (identifier)
(switch_case
(string)
(expression_statement (function_call (identifier) (string)))
(break_statement))
(switch_case
(function_call (identifier))
(expression_statement (function_call (identifier) (string)))
(break_statement))
(switch_case
(expression_statement (function_call (identifier) (string))))))
(switch_statement (identifier)
(switch_case
(string)
(expression_statement (function_call (identifier) (string)))
(break_statement))
(switch_case
(function_call (identifier))
(expression_statement (function_call (identifier) (string)))
(break_statement))
(switch_case
(expression_statement (function_call (identifier) (string)))))

20
spec/runtime/languages/javascript/errors.txt
View file

@ -3,7 +3,7 @@ recovers from errors in function calls
==========================================
stuff(|||);
---
(program (expression_statement (function_call (identifier) (ERROR '|'))))
(expression_statement (function_call (identifier) (ERROR '|')))
==========================================
recovers from errors in if statements
@ -20,21 +20,3 @@ moreStuff();
(statement_block (expression_statement (ERROR '*'))))
(expression_statement (function_call (identifier))))
==========================================
recovers from errors in for loops
==========================================
stuff();
for (var i = 0; *nonsense*; i++) {
*more-nonsense*;
}
moreStuff();
---
(program
(expression_statement (function_call (identifier)))
(for_statement
(var_declaration (identifier) (number))
(expression_statement (ERROR '*'))
(math_op (identifier))
(statement_block (expression_statement (ERROR '*'))))
(expression_statement (function_call (identifier))))

43
spec/runtime/languages/javascript/literals.txt
View file

@ -3,24 +3,24 @@ parses regexes
==========================================
theFunction(/regex1/, /regex2/g);
---
(program (expression_statement (function_call (identifier)
(regex) (regex))))
(expression_statement (function_call (identifier)
(regex) (regex)))
==========================================
parses numbers
==========================================
theFunction(100.0, 200);
---
(program (expression_statement (function_call (identifier)
(number) (number))))
(expression_statement (function_call (identifier)
(number) (number)))
==========================================
parses strings
==========================================
theFunction('', "", 'single-quoted-string', "double-quoted-string");
---
(program (expression_statement (function_call (identifier)
(string) (string) (string) (string))))
(expression_statement (function_call (identifier)
(string) (string) (string) (string)))
==========================================
parses function expressions
@ -31,12 +31,11 @@ var x = {
}
};
---
(program
(var_declaration
(identifier)
(object (identifier) (function_expression
(formal_parameters (identifier) (identifier))
(statement_block (var_declaration (identifier) (identifier)))))))
(var_declaration
(identifier)
(object (identifier) (function_expression
(formal_parameters (identifier) (identifier))
(statement_block (var_declaration (identifier) (identifier))))))
==========================================
parses comments
@ -56,15 +55,14 @@ var thing = {
---
(comment)
(comment)
(program
(var_declaration (identifier) (object
(comment)
(comment)
(identifier) (function_expression
(formal_parameters (identifier) (comment))
(statement_block
(comment)
(expression_statement (function_call (identifier))))))))
(var_declaration (identifier) (object
(comment)
(comment)
(identifier) (function_expression
(formal_parameters (identifier) (comment))
(statement_block
(comment)
(expression_statement (function_call (identifier)))))))
==========================================
parses comments within expressions
@ -72,5 +70,4 @@ parses comments within expressions
y // comment
* z;
---
(program
(expression_statement (math_op (expression (identifier) (comment)) (identifier))))
(expression_statement (math_op (expression (identifier) (comment)) (identifier)))

36
spec/runtime/languages/javascript/operators.txt
View file

@ -3,22 +3,22 @@ parses function calls
==========================================
x.theMethod(5, 6);
---
(program (expression_statement (function_call
(expression_statement (function_call
(property_access (identifier) (identifier))
(number) (number))))
(number) (number)))
==========================================
parses constructor calls
==========================================
var x = new Node(5, new Node(3, null));
---
(program (var_declaration
(var_declaration
(identifier)
(constructor_call (function_call (identifier)
(number)
(constructor_call (function_call (identifier)
(number)
(null)))))))
(null))))))
==========================================
parses property access with dot notation
@ -41,10 +41,10 @@ object
.someProperty
.otherProperty
---
(program (expression_statement
(expression_statement
(property_access
(property_access (identifier) (identifier))
(identifier))))
(identifier)))
===========================================
parses dynamic property access
@ -65,10 +65,10 @@ parses ternary expressions
==========================================
print(isDone() ? stuff : otherStuff);
---
(program (expression_statement
(expression_statement
(function_call
(identifier)
(ternary (function_call (identifier)) (identifier) (identifier)))))
(ternary (function_call (identifier)) (identifier) (identifier))))
==========================================
parses mathematical operators
@ -76,12 +76,12 @@ parses mathematical operators
-a + b * c - d / +e
---
(program (expression_statement
(expression_statement
(math_op
(math_op (identifier))
(math_op
(math_op (identifier) (identifier))
(math_op (identifier) (math_op (identifier)))))))
(math_op (identifier) (math_op (identifier))))))
==========================================
parses boolean operators
@ -89,11 +89,11 @@ parses boolean operators
!a || !(b && c)
---
(program (expression_statement
(expression_statement
(bool_op
(bool_op (identifier))
(bool_op
(expression (bool_op (identifier) (identifier)))))))
(expression (bool_op (identifier) (identifier))))))
===========================================
parses the type operators
@ -101,10 +101,10 @@ parses the type operators
print((x instanceof Array) || (typeof x == "string"))
---
(program (expression_statement (function_call (identifier)
(expression_statement (function_call (identifier)
(bool_op
(expression (instanceof_expression (identifier) (identifier)))
(expression (typeof_expression (bool_op (identifier) (string))))))))
(expression (typeof_expression (bool_op (identifier) (string)))))))
============================================
parses the 'in' operator
@ -112,9 +112,9 @@ parses the 'in' operator
print(x in y)
---
(program (expression_statement (function_call
(expression_statement (function_call
(identifier)
(in_expression (identifier) (identifier)))))
(in_expression (identifier) (identifier))))
============================================
parses assignment operators
@ -136,7 +136,7 @@ parses property access and operators
print(x.y.z && a.b.c)
---
(program (expression_statement (function_call (identifier)
(expression_statement (function_call (identifier)
(bool_op
(property_access (property_access (identifier) (identifier)) (identifier))
(property_access (property_access (identifier) (identifier)) (identifier))))))
(property_access (property_access (identifier) (identifier)) (identifier)))))

10
src/compiler/prepare_grammar/expand_repeats.cc
View file

@ -35,11 +35,11 @@ class ExpandRepeats : public rules::IdentityRuleFn {
string helper_rule_name = rule_name + string("_repeat") + to_string(index);
rule_ptr repeat_symbol =
make_shared<Symbol>(offset + index, rules::SymbolOptionAuxiliary);
aux_rules.push_back(
{ helper_rule_name,
Choice::Build({ Seq::Build({ inner_rule, repeat_symbol }),
make_shared<Blank>() }) });
return repeat_symbol;
aux_rules.push_back({
helper_rule_name,
Seq::Build({ inner_rule, Choice::Build({ repeat_symbol, make_shared<Blank>() }) })
});
return Choice::Build({ repeat_symbol, make_shared<Blank>() });
}
public: