traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Properly incorporate lookahead bytes when recovering via missing token

Browse source
This commit is contained in:
Max Brunsfeld 2022-03-02 16:46:24 -08:00
parent 7170ec7c96
commit 8decec3774
3 changed files with 12 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

13
lib/src/parser.c
View file

@ -1307,9 +1307,11 @@ static void ts_parser__handle_error(
for (StackVersion v = version; v < version_count;) {
if (!did_insert_missing_token) {
TSStateId state = ts_stack_state(self->stack, v);
for (TSSymbol missing_symbol = 1;
missing_symbol < self->language->token_count;
missing_symbol++) {
for (
TSSymbol missing_symbol = 1;
missing_symbol < self->language->token_count;
missing_symbol++
) {
TSStateId state_after_missing_symbol = ts_language_next_state(
self->language, state, missing_symbol
);
@ -1328,10 +1330,13 @@ static void ts_parser__handle_error(
ts_lexer_reset(&self->lexer, position);
ts_lexer_mark_end(&self->lexer);
Length padding = length_sub(self->lexer.token_end_position, position);
uint32_t lookahead_bytes = ts_subtree_total_bytes(lookahead) + ts_subtree_lookahead_bytes(lookahead);
StackVersion version_with_missing_tree = ts_stack_copy_version(self->stack, v);
Subtree missing_tree = ts_subtree_new_missing_leaf(
&self->tree_pool, missing_symbol, padding, self->language
&self->tree_pool, missing_symbol,
padding, lookahead_bytes,
self->language
);
ts_stack_push(
self->stack, version_with_missing_tree,

3
lib/src/subtree.c
View file

@ -559,10 +559,11 @@ Subtree ts_subtree_new_missing_leaf(
SubtreePool *pool,
TSSymbol symbol,
Length padding,
uint32_t lookahead_bytes,
const TSLanguage *language
) {
Subtree result = ts_subtree_new_leaf(
pool, symbol, padding, length_zero(), 0,
pool, symbol, padding, length_zero(), lookahead_bytes,
0, false, false, false, language
);
if (result.data.is_inline) {

2
lib/src/subtree.h
View file

@ -193,7 +193,7 @@ Subtree ts_subtree_new_error(
);
MutableSubtree ts_subtree_new_node(TSSymbol, SubtreeArray *, unsigned, const TSLanguage *);
Subtree ts_subtree_new_error_node(SubtreeArray *, bool, const TSLanguage *);
Subtree ts_subtree_new_missing_leaf(SubtreePool *, TSSymbol, Length, const TSLanguage *);
Subtree ts_subtree_new_missing_leaf(SubtreePool *, TSSymbol, Length, uint32_t, const TSLanguage *);
MutableSubtree ts_subtree_make_mut(SubtreePool *, Subtree);
void ts_subtree_retain(Subtree);
void ts_subtree_release(SubtreePool *, Subtree);