traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Distinguish separators from main tokens via a property on transitions

Browse source 
It was incorrect to store it as a property on the lexical states themselves
This commit is contained in:
Max Brunsfeld 2016-05-19 16:25:44 -07:00
parent 59712ec492
commit a3679fbb1f
13 changed files with 157 additions and 153 deletions
Download patch file Download diff file Expand all files Collapse all files

15
src/runtime/lexer.c
View file

@ -52,20 +52,15 @@ static void ts_lexer__start(TSLexer *self, TSStateId lex_state) {
LOG_LOOKAHEAD();
self->starting_state = lex_state;
self->token_start_position = self->current_position;
if (!self->chunk)
ts_lexer__get_chunk(self);
if (!self->lookahead_size)
ts_lexer__get_lookahead(self);
}
static void ts_lexer__start_token(TSLexer *self) {
LOG("start_token chars:%lu, rows:%lu, columns:%lu",
self->current_position.chars, self->current_position.rows,
self->current_position.columns);
self->token_start_position = self->current_position;
}
static bool ts_lexer__advance(TSLexer *self, TSStateId state) {
static bool ts_lexer__advance(TSLexer *self, TSStateId state,
bool in_main_token) {
LOG("advance state:%d", state);
if (self->chunk == empty_chunk)
@ -83,6 +78,9 @@ static bool ts_lexer__advance(TSLexer *self, TSStateId state) {
}
}
if (!in_main_token)
self->token_start_position = self->current_position;
if (self->current_position.bytes >= self->chunk_start + self->chunk_size)
ts_lexer__get_chunk(self);
@ -125,7 +123,6 @@ static TSTree *ts_lexer__accept(TSLexer *self, TSSymbol symbol,
void ts_lexer_init(TSLexer *self) {
*self = (TSLexer){
.start_fn = ts_lexer__start,
.start_token_fn = ts_lexer__start_token,
.advance_fn = ts_lexer__advance,
.accept_fn = ts_lexer__accept,
.chunk = NULL,

2
src/runtime/stack.c
View file

@ -501,7 +501,7 @@ bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) {
fprintf(f, "rankdir=\"RL\";\n");
fprintf(f, "edge [arrowhead=none]\n");
Array(StackNode *) visited_nodes = array_new();
Array(StackNode *)visited_nodes = array_new();
array_clear(&self->pop_paths);
for (size_t i = 0; i < self->heads.size; i++) {