traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

lexer: in error mode, continue until token is found

Browse source
This commit is contained in:
Max Brunsfeld 2015-06-12 13:13:43 -07:00
parent f7e4445358
commit d5ce3a9b5a
4 changed files with 26 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/runtime/lexer.c
View file

@ -42,6 +42,7 @@ static void read_lookahead(TSLexer *lexer) {
static void start(TSLexer *lexer, TSStateId lex_state) {
DEBUG("start_lex state:%d", lex_state);
DEBUG_LOOKAHEAD();
}
static void start_token(TSLexer *lexer) {
@ -76,7 +77,6 @@ static TSTree *accept(TSLexer *lexer, TSSymbol symbol, int is_hidden,
lexer->token_end_position = lexer->current_position;
if (symbol == ts_builtin_sym_error) {
DEBUG_LOOKAHEAD();
DEBUG("error_char");
return ts_tree_make_error(size, padding, lexer->lookahead);
} else {