traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Build error nodes in lexer again, not in parser

Browse source
This commit is contained in:
Max Brunsfeld 2014-08-31 16:59:01 -07:00
parent 16d5cf1d04
commit 2985a98150
4 changed files with 28 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

28
include/tree_sitter/parser.h
View file

@ -108,16 +108,16 @@ struct TSLanguage {
#define START_TOKEN() ts_lexer_start_token(lexer);
#define ADVANCE(state_index) \
{ \
DEBUG_LEX("ADVANCE %d", state_index); \
if (ts_lexer_is_done(lexer)) { \
DEBUG_LEX("END"); \
return NULL; \
} \
ts_lexer_advance(lexer); \
lex_state = state_index; \
goto next_state; \
#define ADVANCE(state_index) \
{ \
DEBUG_LEX("ADVANCE %d", state_index); \
if (ts_lexer_is_done(lexer)) { \
DEBUG_LEX("END"); \
return ts_lexer_accept(lexer, ts_builtin_sym_error, 0); \
} \
ts_lexer_advance(lexer); \
lex_state = state_index; \
goto next_state; \
}
#define ACCEPT_TOKEN(symbol) \
@ -126,10 +126,10 @@ struct TSLanguage {
return ts_lexer_accept(lexer, symbol, ts_hidden_symbol_flags[symbol]); \
}
#define LEX_ERROR() \
{ \
DEBUG_LEX("ERROR"); \
return NULL; \
#define LEX_ERROR() \
{ \
DEBUG_LEX("ERROR"); \
return ts_lexer_accept(lexer, ts_builtin_sym_error, 0); \
}
#define SHIFT(to_state_value) \

16
spec/runtime/document_spec.cc
View file

@ -134,8 +134,8 @@ describe("Document", [&]() {
TSNode *last = ts_node_child(array, 2);
AssertThat(ts_node_name(error), Equals("error"));
AssertThat(ts_node_pos(error), Equals(string(" [123,").length()))
AssertThat(ts_node_size(error), Equals(string(" @@@@@").length()))
AssertThat(ts_node_pos(error), Equals(string(" [123, ").length()))
AssertThat(ts_node_size(error), Equals(string("@@@@@").length()))
AssertThat(ts_node_name(last), Equals("true"));
AssertThat(ts_node_pos(last), Equals(string(" [123, @@@@@, ").length()))
@ -158,8 +158,8 @@ describe("Document", [&]() {
TSNode *last = ts_node_child(array, 2);
AssertThat(ts_node_name(error), Equals("error"));
AssertThat(ts_node_pos(error), Equals(string(" [123,").length()))
AssertThat(ts_node_size(error), Equals(string(" faaaaalse").length()))
AssertThat(ts_node_pos(error), Equals(string(" [123, ").length()))
AssertThat(ts_node_size(error), Equals(string("faaaaalse").length()))
AssertThat(ts_node_name(last), Equals("true"));
AssertThat(ts_node_pos(last), Equals(string(" [123, faaaaalse, ").length()))
@ -182,8 +182,8 @@ describe("Document", [&]() {
TSNode *last = ts_node_child(array, 2);
AssertThat(ts_node_name(error), Equals("error"));
AssertThat(ts_node_pos(error), Equals(string(" [123,").length()))
AssertThat(ts_node_size(error), Equals(string(" true false").length()))
AssertThat(ts_node_pos(error), Equals(string(" [123, ").length()))
AssertThat(ts_node_size(error), Equals(string("true false").length()))
AssertThat(ts_node_name(last), Equals("true"));
AssertThat(ts_node_pos(last), Equals(string(" [123, true false, ").length()))
@ -206,8 +206,8 @@ describe("Document", [&]() {
TSNode *last = ts_node_child(array, 2);
AssertThat(ts_node_name(error), Equals("error"));
AssertThat(ts_node_pos(error), Equals(string(" [123,").length()))
AssertThat(ts_node_size(error), Equals(string(" ").length()))
AssertThat(ts_node_pos(error), Equals(string(" [123, ").length()))
AssertThat(ts_node_size(error), Equals<size_t>(0))
AssertThat(ts_node_name(last), Equals("true"));
AssertThat(ts_node_pos(last), Equals(string(" [123, , ").length()))

4
src/runtime/lexer.c
View file

@ -26,7 +26,9 @@ static TSTree *accept(TSLexer *lexer, TSSymbol symbol, int is_hidden) {
size_t size = current_position - lexer->token_start_position;
size_t padding = lexer->token_start_position - lexer->token_end_position;
lexer->token_end_position = current_position;
return ts_tree_make_leaf(symbol, size, padding, is_hidden);
return (symbol == ts_builtin_sym_error)
? ts_tree_make_error(size, padding, ts_lexer_lookahead_char(lexer))
: ts_tree_make_leaf(symbol, size, padding, is_hidden);
}
/*

10
src/runtime/parser.c
View file

@ -121,9 +121,6 @@ static void lex(TSParser *parser, TSStateId lex_state) {
if (parser->lookahead)
ts_tree_release(parser->lookahead);
parser->lookahead = parser->language->lex_fn(&parser->lexer, lex_state);
if (!parser->lookahead) {
parser->lookahead = build_error_node(parser);
}
}
static int handle_error(TSParser *parser) {
@ -137,7 +134,7 @@ static int handle_error(TSParser *parser) {
* Unwind the parse stack until a state is found in which an error is
* expected and the current lookahead token is expected afterwards.
*/
size_t error_start_pos = last_token_end;
size_t error_start = last_token_end;
TS_STACK_FROM_TOP(parser->stack, entry, i) {
TSStateId state = entry->state;
TSParseAction action_on_error =
@ -150,8 +147,7 @@ static int handle_error(TSParser *parser) {
if (action_after_error.type != TSParseActionTypeError) {
DEBUG_PARSE("RECOVER %u", state_after_error);
size_t current_position = ts_lexer_position(&parser->lexer);
error->size = current_position - 1 - error_start_pos;
error->size += ts_lexer_position(&parser->lexer) - 1 - error_start;
ts_stack_shrink(&parser->stack, i + 1);
ts_stack_push(&parser->stack, state_after_error, error);
ts_tree_release(error);
@ -160,7 +156,7 @@ static int handle_error(TSParser *parser) {
}
TSTree *removed_tree = entry->node;
error_start_pos -= ts_tree_total_size(removed_tree);
error_start -= ts_tree_total_size(removed_tree);
}
/*