traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refine logic for deciding when tokens need to be re-lexed

Browse source 
* While generating the lex table, note which tokens can match the
  same string. A token needs to be relexed when it has possible
  homonyms in the current state.
* Also note which tokens can match substrings of each other tokens.
  A token needs to be relexed when there are viable tokens that
  could match longer strings in the current state and the next
  token has been edited.
* Remove the logic for marking tokens as fragile on creation.
* Store the reusability/non-reusability of symbols off of individual
  actions and onto the entire entry for the state & symbol.
This commit is contained in:
Max Brunsfeld 2016-06-21 07:28:04 -07:00
parent 45f7cee0c8
commit 38c144b4a3
19 changed files with 337 additions and 257 deletions
Download patch file Download diff file Expand all files Collapse all files

23
src/runtime/tree.c
View file

@ -28,15 +28,13 @@ TSTree *ts_tree_make_leaf(TSSymbol sym, TSLength padding, TSLength size,
.padding = padding,
.visible = metadata.visible,
.named = metadata.named,
.lex_state = TS_TREE_STATE_INDEPENDENT,
.parse_state = TS_TREE_STATE_INDEPENDENT,
.first_leaf =
{
.symbol = sym, .lex_state = TS_TREE_STATE_INDEPENDENT,
},
};
if (sym == ts_builtin_sym_error) {
result->fragile_left = true;
result->fragile_right = true;
}
return result;
}
@ -81,6 +79,8 @@ TSTree *ts_tree_make_error(TSLength size, TSLength padding, char lookahead_char)
if (!result)
return NULL;
result->fragile_left = true;
result->fragile_right = true;
result->lookahead_char = lookahead_char;
return result;
}
@ -174,7 +174,7 @@ void ts_tree_set_children(TSTree *self, size_t child_count, TSTree **children) {
}
if (child_count > 0) {
self->lex_state = children[0]->lex_state;
self->first_leaf = children[0]->first_leaf;
if (children[0]->fragile_left)
self->fragile_left = true;
if (children[child_count - 1]->fragile_right)
@ -206,9 +206,16 @@ TSTree *ts_tree_make_error_node(TreeArray *children) {
}
}
return ts_tree_make_node(
TSTree *result = ts_tree_make_node(
ts_builtin_sym_error, children->size, children->contents,
(TSSymbolMetadata){.extra = false, .visible = true, .named = true });
if (!result)
return NULL;
result->fragile_left = true;
result->fragile_right = true;
return result;
}
void ts_tree_retain(TSTree *self) {