traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Control lexer's error-mode via explicit boolean argument

Browse source 
Previously, the lexer would operate in error-mode (ignoring any garbage input
until it found a valid token) if it was invoked in the 'error' state. Now that
the error state is deduped with other lexical states, the lexer might be invoked
in that state even when error-mode is not intended. This adds a third argument
to `ts_lex` that explicitly sets the error-mode.

This bug was unlikely to occur in any real grammars, but it caused the
node-tree-sitter-compiler test suite to fail for some grammars with only one
rule.
This commit is contained in:
Max Brunsfeld 2015-12-30 09:37:40 -08:00
parent 4ad1a666be
commit 4b04afac5e
10 changed files with 442 additions and 452 deletions
Download patch file Download diff file Expand all files Collapse all files

37
include/tree_sitter/parser.h
View file

@ -87,7 +87,7 @@ struct TSLanguage {
const unsigned short *parse_table;
const TSParseActionEntry *parse_actions;
const TSStateId *lex_states;
TSTree *(*lex_fn)(TSLexer *, TSStateId);
TSTree *(*lex_fn)(TSLexer *, TSStateId, bool);
};
/*
@ -95,24 +95,23 @@ struct TSLanguage {
*/
#define START_LEXER() \
const bool error_mode = (lex_state == ts_lex_state_error); \
lexer->start_fn(lexer, lex_state); \
lexer->start_fn(lexer, state); \
int32_t lookahead; \
next_state: \
lookahead = lexer->lookahead;
#define START_TOKEN() lexer->start_token_fn(lexer);
#define GO_TO_STATE(state_index) \
#define GO_TO_STATE(state_value) \
{ \
lex_state = state_index; \
state = state_value; \
goto next_state; \
}
#define ADVANCE(state_index) \
#define ADVANCE(state_value) \
{ \
lexer->advance_fn(lexer, state_index); \
GO_TO_STATE(state_index); \
lexer->advance_fn(lexer, state_value); \
GO_TO_STATE(state_value); \
}
#define ACCEPT_FRAGILE_TOKEN(symbol) \
@ -123,27 +122,19 @@ struct TSLanguage {
return lexer->accept_fn(lexer, symbol, ts_symbol_metadata[symbol], \
ts_symbol_names[symbol], false);
#define LEX_ERROR() \
if (error_mode) { \
if (lex_state == ts_lex_state_error) \
ADVANCE(ts_lex_state_error) \
else \
GO_TO_STATE(ts_lex_state_error) \
} else { \
ACCEPT_TOKEN(ts_builtin_sym_error) \
#define LEX_ERROR() \
if (error_mode) { \
if (state == ts_lex_state_error) \
lexer->advance_fn(lexer, state); \
GO_TO_STATE(ts_lex_state_error) \
} else { \
ACCEPT_TOKEN(ts_builtin_sym_error) \
}
/*
* Parse Table Macros
*/
#define ACTIONS(...) \
(TSParseAction[]) { \
__VA_ARGS__, { \
.type = 0 \
} \
}
enum {
FRAGILE = 1,
CAN_HIDE_SPLIT = 2,