traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Allow lexer to accept tokens that ended at previous positions

Browse source 
* Track lookahead in each tree
* Add 'mark_end' API that external scanners can use
This commit is contained in:
Max Brunsfeld 2017-03-13 17:03:47 -07:00
parent 12d2a9d93f
commit d222dbb9fd
12 changed files with 96 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

35
include/tree_sitter/parser.h
View file

@ -26,6 +26,7 @@ typedef struct {
typedef struct {
void (*advance)(void *, bool);
void (*mark_end)(void *);
int32_t lookahead;
TSSymbol result_symbol;
} TSLexer;
@ -91,32 +92,32 @@ typedef struct TSLanguage {
* Lexer Macros
*/
#define START_LEXER() \
int32_t lookahead; \
next_state: \
#define START_LEXER() \
bool result = false; \
int32_t lookahead; \
next_state: \
lookahead = lexer->lookahead;
#define ADVANCE(state_value) \
{ \
#define ADVANCE(state_value) \
{ \
lexer->advance(lexer, false); \
state = state_value; \
goto next_state; \
state = state_value; \
goto next_state; \
}
#define SKIP(state_value) \
{ \
#define SKIP(state_value) \
{ \
lexer->advance(lexer, true); \
state = state_value; \
goto next_state; \
state = state_value; \
goto next_state; \
}
#define ACCEPT_TOKEN(symbol_value) \
{ \
lexer->result_symbol = symbol_value; \
return true; \
}
#define ACCEPT_TOKEN(symbol_value) \
result = true; \
lexer->result_symbol = symbol_value; \
lexer->mark_end(lexer);
#define LEX_ERROR() return false
#define END_STATE() return result;
/*
* Parse Table Macros

2
include/tree_sitter/runtime.h
View file

@ -9,7 +9,7 @@ extern "C" {
#include <stdint.h>
#include <stdbool.h>
#define TREE_SITTER_LANGUAGE_VERSION 1
#define TREE_SITTER_LANGUAGE_VERSION 2
typedef unsigned short TSSymbol;
typedef struct TSLanguage TSLanguage;