diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h index b09d7d58..abfad34b 100644 --- a/include/tree_sitter/parser.h +++ b/include/tree_sitter/parser.h @@ -49,16 +49,29 @@ DEBUG_LEX("CHAR '%c'", lookahead); ts_lexer_start_token(lexer); #define ADVANCE(state_index) \ -{ DEBUG_LEX("ADVANCE %d", state_index); ts_lexer_advance(lexer); lex_state = state_index; goto next_state; } +{ \ + DEBUG_LEX("ADVANCE %d", state_index); \ + if (!ts_lexer_advance(lexer)) ACCEPT_TOKEN(ts_builtin_sym_end); \ + lex_state = state_index; goto next_state; \ +} #define ACCEPT_TOKEN(symbol) \ -{ DEBUG_LEX("TOKEN %s", ts_symbol_names[symbol]); return ts_lexer_build_node(lexer, symbol); } +{ \ + DEBUG_LEX("TOKEN %s", ts_symbol_names[symbol]); \ + return ts_lexer_build_node(lexer, symbol); \ +} #define LEX_ERROR() \ -{ DEBUG_LEX("ERROR"); return ts_lexer_build_node(lexer, ts_builtin_sym_error); } +{ \ + DEBUG_LEX("ERROR"); \ + return ts_lexer_build_node(lexer, ts_builtin_sym_error); \ +} #define LEX_PANIC() \ -{ DEBUG_LEX("LEX ERROR: unexpected state %d", lex_state); return NULL; } +{ \ + DEBUG_LEX("LEX ERROR: unexpected state %d", lex_state); \ + return NULL; \ +} SYMBOL_NAMES; diff --git a/include/tree_sitter/parser/lexer.h b/include/tree_sitter/parser/lexer.h index 814d4429..3a99fef8 100644 --- a/include/tree_sitter/parser/lexer.h +++ b/include/tree_sitter/parser/lexer.h @@ -37,20 +37,20 @@ static inline char ts_lexer_lookahead_char(const ts_lexer *lexer) { } static inline int ts_lexer_advance(ts_lexer *lexer) { - static const char empty_chunk[1] = ""; + static const char *empty_chunk = ""; if (lexer->position_in_chunk + 1 < lexer->chunk_size) { lexer->position_in_chunk++; } else { lexer->chunk_start += lexer->chunk_size; lexer->chunk = lexer->input.read_fn(lexer->input.data, &lexer->chunk_size); - if (lexer->chunk_size == 0) { - if (lexer->reached_end) - return 0; - lexer->chunk = empty_chunk; - lexer->chunk_size = 1; - lexer->reached_end = 1; - } lexer->position_in_chunk = 0; + if (lexer->reached_end) { + return 0; + } + if (lexer->chunk_size == 0) { + lexer->reached_end = 1; + lexer->chunk = empty_chunk; + } } return 1; }