traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

In lexer, prefer tokens to skipped separator characters

Browse source 
This was causing newlines in go and javascript to be parsed as
meaningless separator characters instead of statement terminators
This commit is contained in:
Max Brunsfeld 2014-05-30 13:29:54 -07:00
parent 220e081c49
commit e93e254518
26 changed files with 5559 additions and 6650 deletions
Download patch file Download diff file Expand all files Collapse all files

3
include/tree_sitter/parser.h
View file

@ -31,6 +31,7 @@ static ts_state_id ts_lex_states[STATE_COUNT]
static ts_tree * ts_lex(ts_lexer *lexer, ts_state_id lex_state)
#define START_LEXER() \
DEBUG_LEX("LEX %d", lex_state); \
char lookahead; \
next_state: \
lookahead = ts_lexer_lookahead_char(lexer); \
@ -46,7 +47,7 @@ ts_lexer_start_token(lexer);
{ DEBUG_LEX("TOKEN %s", ts_symbol_names[symbol]); return ts_lexer_build_node(lexer, symbol); }
#define LEX_ERROR() \
{ return ts_lexer_build_node(lexer, ts_builtin_sym_error); }
{ DEBUG_LEX("ERROR"); return ts_lexer_build_node(lexer, ts_builtin_sym_error); }
#define LEX_PANIC() \
{ DEBUG_LEX("LEX ERROR: unexpected state %d", lex_state); return NULL; }