traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Distinguish separators from main tokens via a property on transitions

Browse source 
It was incorrect to store it as a property on the lexical states themselves
This commit is contained in:
Max Brunsfeld 2016-05-19 16:25:44 -07:00
parent 59712ec492
commit a3679fbb1f
13 changed files with 157 additions and 153 deletions
Download patch file Download diff file Expand all files Collapse all files

21
include/tree_sitter/parser.h
View file

@ -33,8 +33,7 @@ typedef struct {
typedef struct TSLexer {
void (*start_fn)(struct TSLexer *, TSStateId);
void (*start_token_fn)(struct TSLexer *);
bool (*advance_fn)(struct TSLexer *, TSStateId);
bool (*advance_fn)(struct TSLexer *, TSStateId, bool);
TSTree *(*accept_fn)(struct TSLexer *, TSSymbol, TSSymbolMetadata,
const char *, bool fragile);
@ -103,18 +102,22 @@ struct TSLanguage {
next_state: \
lookahead = lexer->lookahead;
#define START_TOKEN() lexer->start_token_fn(lexer);
#define GO_TO_STATE(state_value) \
{ \
state = state_value; \
goto next_state; \
}
#define ADVANCE(state_value) \
{ \
lexer->advance_fn(lexer, state_value); \
GO_TO_STATE(state_value); \
#define ADVANCE(state_value) \
{ \
lexer->advance_fn(lexer, state_value, true); \
GO_TO_STATE(state_value); \
}
#define SKIP(state_value) \
{ \
lexer->advance_fn(lexer, state_value, false); \
GO_TO_STATE(state_value); \
}
#define ACCEPT_FRAGILE_TOKEN(symbol) \
@ -128,7 +131,7 @@ struct TSLanguage {
#define LEX_ERROR() \
if (error_mode) { \
if (state == ts_lex_state_error) \
lexer->advance_fn(lexer, state); \
lexer->advance_fn(lexer, state, true); \
GO_TO_STATE(ts_lex_state_error) \
} else { \
return lexer->accept_fn(lexer, ts_builtin_sym_error, (TSSymbolMetadata){}, \