traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix missing auxiliary tokens in lex error state

Browse source
This commit is contained in:
Max Brunsfeld 2014-02-27 00:38:08 -08:00
parent 926b4ef242
commit c43ec90dad
5 changed files with 81 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

2
examples/grammars/arithmetic.hpp
View file

@ -26,7 +26,7 @@ namespace test_grammars {
sym("number"),
seq({
str("("),
sym("expression"),
err(sym("expression")),
str(")") }) }) },
{ "plus", str("+") },
{ "times", str("*") },

52
examples/parsers/arithmetic.c
View file

@ -26,6 +26,22 @@ SYMBOL_NAMES {
static const ts_symbol * ts_recover(ts_state state, ts_state *to_state, size_t *count) {
switch (state) {
case 6:
RECOVER(7, 1, EXPECT({ts_aux_sym_token2}));
case 13:
RECOVER(14, 1, EXPECT({ts_aux_sym_token2}));
case 23:
RECOVER(24, 1, EXPECT({ts_aux_sym_token2}));
case 26:
RECOVER(27, 1, EXPECT({ts_aux_sym_token2}));
case 31:
RECOVER(32, 1, EXPECT({ts_aux_sym_token2}));
case 41:
RECOVER(42, 1, EXPECT({ts_aux_sym_token2}));
case 46:
RECOVER(47, 1, EXPECT({ts_aux_sym_token2}));
case 49:
RECOVER(50, 1, EXPECT({ts_aux_sym_token2}));
default:
RECOVER(0, 0, EXPECT({}));
}
@ -101,6 +117,10 @@ LEX_FN() {
ADVANCE(7);
LEX_ERROR();
case ts_lex_state_error:
if (LOOKAHEAD_CHAR() == '(')
ADVANCE(9);
if (LOOKAHEAD_CHAR() == ')')
ADVANCE(4);
if (LOOKAHEAD_CHAR() == '*')
ADVANCE(7);
if (LOOKAHEAD_CHAR() == '+')
@ -206,8 +226,10 @@ PARSE_FN() {
SHIFT(16);
case ts_aux_sym_token1:
SHIFT(26);
case ts_builtin_sym_error:
SHIFT(7);
default:
PARSE_ERROR(6, EXPECT({ts_sym_expression, ts_sym_factor, ts_sym_number, ts_sym_term, ts_sym_variable, ts_aux_sym_token1}));
PARSE_ERROR(7, EXPECT({ts_sym_expression, ts_sym_factor, ts_sym_number, ts_sym_term, ts_sym_variable, ts_aux_sym_token1, ts_builtin_sym_error}));
}
case 7:
SET_LEX_STATE(3);
@ -288,8 +310,10 @@ PARSE_FN() {
SHIFT(16);
case ts_aux_sym_token1:
SHIFT(26);
case ts_builtin_sym_error:
SHIFT(14);
default:
PARSE_ERROR(6, EXPECT({ts_sym_expression, ts_sym_factor, ts_sym_number, ts_sym_term, ts_sym_variable, ts_aux_sym_token1}));
PARSE_ERROR(7, EXPECT({ts_sym_expression, ts_sym_factor, ts_sym_number, ts_sym_term, ts_sym_variable, ts_aux_sym_token1, ts_builtin_sym_error}));
}
case 14:
SET_LEX_STATE(3);
@ -402,8 +426,10 @@ PARSE_FN() {
SHIFT(16);
case ts_aux_sym_token1:
SHIFT(26);
case ts_builtin_sym_error:
SHIFT(24);
default:
PARSE_ERROR(6, EXPECT({ts_sym_expression, ts_sym_factor, ts_sym_number, ts_sym_term, ts_sym_variable, ts_aux_sym_token1}));
PARSE_ERROR(7, EXPECT({ts_sym_expression, ts_sym_factor, ts_sym_number, ts_sym_term, ts_sym_variable, ts_aux_sym_token1, ts_builtin_sym_error}));
}
case 24:
SET_LEX_STATE(3);
@ -436,8 +462,10 @@ PARSE_FN() {
SHIFT(16);
case ts_aux_sym_token1:
SHIFT(26);
case ts_builtin_sym_error:
SHIFT(27);
default:
PARSE_ERROR(6, EXPECT({ts_sym_expression, ts_sym_factor, ts_sym_number, ts_sym_term, ts_sym_variable, ts_aux_sym_token1}));
PARSE_ERROR(7, EXPECT({ts_sym_expression, ts_sym_factor, ts_sym_number, ts_sym_term, ts_sym_variable, ts_aux_sym_token1, ts_builtin_sym_error}));
}
case 27:
SET_LEX_STATE(3);
@ -492,8 +520,10 @@ PARSE_FN() {
SHIFT(16);
case ts_aux_sym_token1:
SHIFT(26);
case ts_builtin_sym_error:
SHIFT(32);
default:
PARSE_ERROR(6, EXPECT({ts_sym_expression, ts_sym_factor, ts_sym_number, ts_sym_term, ts_sym_variable, ts_aux_sym_token1}));
PARSE_ERROR(7, EXPECT({ts_sym_expression, ts_sym_factor, ts_sym_number, ts_sym_term, ts_sym_variable, ts_aux_sym_token1, ts_builtin_sym_error}));
}
case 32:
SET_LEX_STATE(3);
@ -606,8 +636,10 @@ PARSE_FN() {
SHIFT(16);
case ts_aux_sym_token1:
SHIFT(26);
case ts_builtin_sym_error:
SHIFT(42);
default:
PARSE_ERROR(6, EXPECT({ts_sym_expression, ts_sym_factor, ts_sym_number, ts_sym_term, ts_sym_variable, ts_aux_sym_token1}));
PARSE_ERROR(7, EXPECT({ts_sym_expression, ts_sym_factor, ts_sym_number, ts_sym_term, ts_sym_variable, ts_aux_sym_token1, ts_builtin_sym_error}));
}
case 42:
SET_LEX_STATE(3);
@ -658,8 +690,10 @@ PARSE_FN() {
SHIFT(16);
case ts_aux_sym_token1:
SHIFT(26);
case ts_builtin_sym_error:
SHIFT(47);
default:
PARSE_ERROR(6, EXPECT({ts_sym_expression, ts_sym_factor, ts_sym_number, ts_sym_term, ts_sym_variable, ts_aux_sym_token1}));
PARSE_ERROR(7, EXPECT({ts_sym_expression, ts_sym_factor, ts_sym_number, ts_sym_term, ts_sym_variable, ts_aux_sym_token1, ts_builtin_sym_error}));
}
case 47:
SET_LEX_STATE(3);
@ -694,8 +728,10 @@ PARSE_FN() {
SHIFT(16);
case ts_aux_sym_token1:
SHIFT(26);
case ts_builtin_sym_error:
SHIFT(50);
default:
PARSE_ERROR(6, EXPECT({ts_sym_expression, ts_sym_factor, ts_sym_number, ts_sym_term, ts_sym_variable, ts_aux_sym_token1}));
PARSE_ERROR(7, EXPECT({ts_sym_expression, ts_sym_factor, ts_sym_number, ts_sym_term, ts_sym_variable, ts_aux_sym_token1, ts_builtin_sym_error}));
}
case 50:
SET_LEX_STATE(3);

2
spec/main.cpp
View file

@ -6,7 +6,7 @@ int main(int argc, char *argv[])
"",
"--no-color",
"--only="
""
"parenthesized expr"
};
return bandit::run(4, const_cast<char **>(args));
}

47
spec/runtime/arithmetic_spec.cpp
View file

@ -5,44 +5,61 @@ extern ts_parse_config ts_parse_config_arithmetic;
START_TEST
describe("arithmetic", []() {
ts_document *document;
ts_document *doc;
before_each([&]() {
document = ts_document_make();
ts_document_set_parser(document, ts_parse_config_arithmetic);
doc = ts_document_make();
ts_document_set_parser(doc, ts_parse_config_arithmetic);
});
after_each([&]() {
ts_document_free(doc);
});
it("parses variables", [&]() {
ts_document_set_text(document, "x");
AssertThat(string(ts_document_string(document)), Equals(
ts_document_set_text(doc, "x");
AssertThat(string(ts_document_string(doc)), Equals(
"(expression (term (factor (variable))))"));
});
it("parses numbers", [&]() {
ts_document_set_text(document, "5");
AssertThat(string(ts_document_string(document)), Equals(
ts_document_set_text(doc, "5");
AssertThat(string(ts_document_string(doc)), Equals(
"(expression (term (factor (number))))"));
});
it("parses products of variables", [&]() {
ts_document_set_text(document, "x + y");
AssertThat(string(ts_document_string(document)), Equals(
ts_document_set_text(doc, "x + y");
AssertThat(string(ts_document_string(doc)), Equals(
"(expression (term (factor (variable))) (plus) (term (factor (variable))))"));
ts_document_set_text(document, "x * y");
AssertThat(string(ts_document_string(document)), Equals(
ts_document_set_text(doc, "x * y");
AssertThat(string(ts_document_string(doc)), Equals(
"(expression (term (factor (variable)) (times) (factor (variable))))"));
});
it("parses complex trees", [&]() {
ts_document_set_text(document, "x * y + z * a");
AssertThat(string(ts_document_string(document)), Equals(
ts_document_set_text(doc, "x * y + z * a");
AssertThat(string(ts_document_string(doc)), Equals(
"(expression (term (factor (variable)) (times) (factor (variable))) (plus) (term (factor (variable)) (times) (factor (variable))))"));
ts_document_set_text(document, "x * (y + z)");
AssertThat(string(ts_document_string(document)), Equals(
ts_document_set_text(doc, "x * (y + z)");
AssertThat(string(ts_document_string(doc)), Equals(
"(expression (term (factor (variable)) (times) (factor (expression (term (factor (variable))) (plus) (term (factor (variable)))))))"));
});
describe("error recovery", [&]() {
it("recovers from errors at the top level", [&]() {
ts_document_set_text(doc, "x * * y");
AssertThat(string(ts_document_string(doc)), Equals("(ERROR)"));
});
it("recovers from errors in parenthesized expressions", [&]() {
ts_document_set_text(doc, "x + (y * + z) * 5");
AssertThat(string(ts_document_string(doc)), Equals(
"(expression (term (factor (variable))) (plus) (term (factor (ERROR)) (times) (factor (number))))"));
});
});
});
END_TEST

4
src/compiler/build_tables/perform.cpp
View file

@ -115,7 +115,9 @@ namespace tree_sitter {
void add_error_lex_state() {
LexItemSet error_item_set;
for (auto &pair : lex_grammar.rules)
error_item_set.insert(LexItem(pair.first, pair.second));
error_item_set.insert(LexItem(Symbol(pair.first), pair.second));
for (auto &pair : lex_grammar.aux_rules)
error_item_set.insert(LexItem(Symbol(pair.first, rules::SymbolTypeAuxiliary), pair.second));
add_advance_actions(error_item_set, LexTable::ERROR_STATE_ID);
add_accept_token_actions(error_item_set, LexTable::ERROR_STATE_ID);
}