traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Cram terminal subtree data into a 64-bit integer when possible

Browse source
This commit is contained in:
Max Brunsfeld 2018-09-17 13:12:13 -07:00
parent e00c3bbdb9
commit b29d0f622f
21 changed files with 1258 additions and 1007 deletions
Download patch file Download diff file Expand all files Collapse all files

402
src/runtime/parser.c
View file

@ -35,14 +35,16 @@
#define SYM_NAME(symbol) ts_language_symbol_name(self->language, symbol)
#define TREE_NAME(tree) SYM_NAME(ts_subtree_symbol(tree))
static const unsigned MAX_VERSION_COUNT = 6;
static const unsigned MAX_VERSION_COUNT_OVERFLOW = 4;
static const unsigned MAX_SUMMARY_DEPTH = 16;
static const unsigned MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE;
typedef struct {
const Subtree *token;
const Subtree *last_external_token;
Subtree token;
Subtree last_external_token;
uint32_t byte_index;
} TokenCache;
@ -52,8 +54,9 @@ struct TSParser {
SubtreePool tree_pool;
const TSLanguage *language;
ReduceActionSet reduce_actions;
const Subtree *finished_tree;
Subtree scratch_tree;
Subtree finished_tree;
SubtreeHeapData scratch_tree_data;
MutableSubtree scratch_tree;
TokenCache token_cache;
ReusableNode reusable_node;
void *external_scanner_payload;
@ -62,7 +65,7 @@ struct TSParser {
size_t operation_limit;
volatile bool enabled;
bool halt_on_error;
const Subtree *old_tree;
Subtree old_tree;
};
typedef struct {
@ -132,16 +135,16 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion versi
for (uint32_t i = 0; i < pop.size; i++) {
StackSlice slice = pop.contents[i];
TSStateId state = ts_stack_state(self->stack, slice.version);
const Subtree *parent = *array_front(&slice.subtrees);
Subtree parent = *array_front(&slice.subtrees);
for (uint32_t j = 0; j < parent->child_count; j++) {
const Subtree *child = parent->children[j];
pending = child->child_count > 0;
for (uint32_t j = 0, n = ts_subtree_child_count(parent); j < n; j++) {
Subtree child = parent.ptr->children[j];
pending = ts_subtree_child_count(child) > 0;
if (child->symbol == ts_builtin_sym_error) {
if (ts_subtree_is_error(child)) {
state = ERROR_STATE;
} else if (!child->extra) {
state = ts_language_next_state(self->language, state, child->symbol);
} else if (!ts_subtree_extra(child)) {
state = ts_language_next_state(self->language, state, ts_subtree_symbol(child));
}
ts_subtree_retain(child);
@ -149,14 +152,14 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion versi
}
for (uint32_t j = 1; j < slice.subtrees.size; j++) {
const Subtree *tree = slice.subtrees.contents[j];
Subtree tree = slice.subtrees.contents[j];
ts_stack_push(self->stack, slice.version, tree, false, state);
}
ts_subtree_release(&self->tree_pool, parent);
array_delete(&slice.subtrees);
LOG("breakdown_top_of_stack tree:%s", SYM_NAME(parent->symbol));
LOG("breakdown_top_of_stack tree:%s", TREE_NAME(parent));
LOG_STACK();
}
} while (pending);
@ -164,12 +167,12 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion versi
return did_break_down;
}
static void ts_parser__breakdown_lookahead(TSParser *self, const Subtree **lookahead,
static void ts_parser__breakdown_lookahead(TSParser *self, Subtree *lookahead,
TSStateId state, ReusableNode *reusable_node) {
bool did_descend = false;
const Subtree *tree = reusable_node_tree(reusable_node);
while (tree->child_count > 0 && tree->parse_state != state) {
LOG("state_mismatch sym:%s", SYM_NAME(tree->symbol));
Subtree tree = reusable_node_tree(reusable_node);
while (ts_subtree_child_count(tree) > 0 && ts_subtree_parse_state(tree) != state) {
LOG("state_mismatch sym:%s", TREE_NAME(tree));
reusable_node_descend(reusable_node);
tree = reusable_node_tree(reusable_node);
did_descend = true;
@ -233,8 +236,10 @@ static ErrorStatus ts_parser__version_status(TSParser *self, StackVersion versio
}
static bool ts_parser__better_version_exists(TSParser *self, StackVersion version,
bool is_in_error, unsigned cost) {
if (self->finished_tree && self->finished_tree->error_cost <= cost) return true;
bool is_in_error, unsigned cost) {
if (self->finished_tree.ptr && ts_subtree_error_cost(self->finished_tree) <= cost) {
return true;
}
Length position = ts_stack_position(self->stack, version);
ErrorStatus status = {
@ -262,50 +267,41 @@ static bool ts_parser__better_version_exists(TSParser *self, StackVersion versio
return false;
}
static void ts_parser__restore_external_scanner(TSParser *self, const Subtree *external_token) {
if (external_token) {
static void ts_parser__restore_external_scanner(TSParser *self, Subtree external_token) {
if (external_token.ptr) {
self->language->external_scanner.deserialize(
self->external_scanner_payload,
ts_external_scanner_state_data(&external_token->external_scanner_state),
external_token->external_scanner_state.length
ts_external_scanner_state_data(&external_token.ptr->external_scanner_state),
external_token.ptr->external_scanner_state.length
);
} else {
self->language->external_scanner.deserialize(self->external_scanner_payload, NULL, 0);
}
}
static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, const Subtree *tree,
static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, Subtree tree,
TableEntry *table_entry) {
TSLexMode current_lex_mode = self->language->lex_modes[state];
TSStateId leaf_state;
TSSymbol leaf_symbol;
if (tree->child_count > 0) {
leaf_state = tree->first_leaf.parse_state;
leaf_symbol = tree->first_leaf.symbol;
} else {
leaf_state = tree->parse_state;
leaf_symbol = tree->symbol;
}
TSSymbol leaf_symbol = ts_subtree_leaf_symbol(tree);
TSStateId leaf_state = ts_subtree_leaf_parse_state(tree);
TSLexMode leaf_lex_mode = self->language->lex_modes[leaf_state];
// If the token was created in a state with the same set of lookaheads, it is reusable.
if (memcmp(&leaf_lex_mode, &current_lex_mode, sizeof(TSLexMode)) == 0 &&
(leaf_symbol != self->language->keyword_capture_token ||
(!tree->is_keyword && tree->parse_state == state))) return true;
(!ts_subtree_is_keyword(tree) && ts_subtree_parse_state(tree) == state))) return true;
// Empty tokens are not reusable in states with different lookaheads.
if (tree->size.bytes == 0 && tree->symbol != ts_builtin_sym_end) return false;
if (ts_subtree_size(tree).bytes == 0 && leaf_symbol != ts_builtin_sym_end) return false;
// If the current state allows external tokens or other tokens that conflict with this
// token, this token is not reusable.
return current_lex_mode.external_lex_state == 0 && table_entry->is_reusable;
}
static const Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSStateId parse_state) {
static Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId parse_state) {
Length start_position = ts_stack_position(self->stack, version);
const Subtree *external_token = ts_stack_last_external_token(self->stack, version);
Subtree external_token = ts_stack_last_external_token(self->stack, version);
TSLexMode lex_mode = self->language->lex_modes[parse_state];
const bool *valid_external_tokens = ts_language_enabled_external_tokens(
self->language,
@ -404,13 +400,19 @@ static const Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSSta
uint32_t bytes_scanned = last_byte_scanned - start_position.bytes + 1;
Subtree *result;
Subtree result;
if (skipped_error) {
Length padding = length_sub(error_start_position, start_position);
Length size = length_sub(error_end_position, error_start_position);
result = ts_subtree_new_error(&self->tree_pool, size, padding, first_error_character, self->language);
result->parse_state = parse_state;
result->bytes_scanned = bytes_scanned;
result = ts_subtree_new_error(
&self->tree_pool,
first_error_character,
padding,
size,
bytes_scanned,
parse_state,
self->language
);
} else {
if (self->lexer.token_end_position.bytes < self->lexer.token_start_position.bytes) {
self->lexer.token_start_position = self->lexer.token_end_position;
@ -455,85 +457,87 @@ static const Subtree *ts_parser__lex(TSParser *self, StackVersion version, TSSta
self->lexer.debug_buffer
);
ts_external_scanner_state_init(
&result->external_scanner_state,
&((SubtreeHeapData *)result.ptr)->external_scanner_state,
self->lexer.debug_buffer,
length
);
}
}
LOG("lexed_lookahead sym:%s, size:%u", SYM_NAME(result->symbol), result->size.bytes);
LOG(
"lexed_lookahead sym:%s, size:%u",
SYM_NAME(ts_subtree_symbol(result)),
ts_subtree_size(result).bytes
);
return result;
}
static const Subtree *ts_parser__get_cached_token(TSParser *self, TSStateId state,
size_t position,
const Subtree *last_external_token,
TableEntry *table_entry) {
static Subtree ts_parser__get_cached_token(TSParser *self, TSStateId state,
size_t position, Subtree last_external_token,
TableEntry *table_entry) {
TokenCache *cache = &self->token_cache;
if (
cache->token && cache->byte_index == position &&
cache->token.ptr && cache->byte_index == position &&
ts_subtree_external_scanner_state_eq(cache->last_external_token, last_external_token)
) {
ts_language_table_entry(self->language, state, cache->token->symbol, table_entry);
ts_language_table_entry(self->language, state, ts_subtree_symbol(cache->token), table_entry);
if (ts_parser__can_reuse_first_leaf(self, state, cache->token, table_entry)) {
ts_subtree_retain(cache->token);
return cache->token;
}
}
return NULL;
return NULL_SUBTREE;
}
static void ts_parser__set_cached_token(TSParser *self, size_t byte_index,
const Subtree *last_external_token,
const Subtree *token) {
Subtree last_external_token,
Subtree token) {
TokenCache *cache = &self->token_cache;
if (token) ts_subtree_retain(token);
if (last_external_token) ts_subtree_retain(last_external_token);
if (cache->token) ts_subtree_release(&self->tree_pool, cache->token);
if (cache->last_external_token) ts_subtree_release(&self->tree_pool, cache->last_external_token);
if (token.ptr) ts_subtree_retain(token);
if (last_external_token.ptr) ts_subtree_retain(last_external_token);
if (cache->token.ptr) ts_subtree_release(&self->tree_pool, cache->token);
if (cache->last_external_token.ptr) ts_subtree_release(&self->tree_pool, cache->last_external_token);
cache->token = token;
cache->byte_index = byte_index;
cache->last_external_token = last_external_token;
}
static const Subtree *ts_parser__reuse_node(TSParser *self, StackVersion version,
TSStateId *state, uint32_t position,
const Subtree *last_external_token,
TableEntry *table_entry) {
const Subtree *result;
while ((result = reusable_node_tree(&self->reusable_node))) {
static Subtree ts_parser__reuse_node(TSParser *self, StackVersion version,
TSStateId *state, uint32_t position,
Subtree last_external_token, TableEntry *table_entry) {
Subtree result;
while ((result = reusable_node_tree(&self->reusable_node)).ptr) {
uint32_t byte_offset = reusable_node_byte_offset(&self->reusable_node);
if (byte_offset > position) {
LOG("before_reusable_node symbol:%s", SYM_NAME(result->symbol));
LOG("before_reusable_node symbol:%s", TREE_NAME(result));
break;
}
if (byte_offset < position) {
LOG("past_reusable_node symbol:%s", SYM_NAME(result->symbol));
LOG("past_reusable_node symbol:%s", TREE_NAME(result));
reusable_node_advance(&self->reusable_node);
continue;
}
if (!ts_subtree_external_scanner_state_eq(self->reusable_node.last_external_token, last_external_token)) {
LOG("reusable_node_has_different_external_scanner_state symbol:%s", SYM_NAME(result->symbol));
LOG("reusable_node_has_different_external_scanner_state symbol:%s", TREE_NAME(result));
reusable_node_advance(&self->reusable_node);
continue;
}
const char *reason = NULL;
if (result->has_changes) {
if (ts_subtree_has_changes(result)) {
reason = "has_changes";
} else if (result->symbol == ts_builtin_sym_error) {
} else if (ts_subtree_is_error(result)) {
reason = "is_error";
} else if (result->is_missing) {
} else if (ts_subtree_missing(result)) {
reason = "is_missing";
} else if (result->fragile_left || result->fragile_right) {
} else if (ts_subtree_is_fragile(result)) {
reason = "is_fragile";
}
if (reason) {
LOG("cant_reuse_node_%s tree:%s", reason, SYM_NAME(result->symbol));
LOG("cant_reuse_node_%s tree:%s", reason, TREE_NAME(result));
if (!reusable_node_descend(&self->reusable_node)) {
reusable_node_advance(&self->reusable_node);
ts_parser__breakdown_top_of_stack(self, version);
@ -547,97 +551,92 @@ static const Subtree *ts_parser__reuse_node(TSParser *self, StackVersion version
if (!ts_parser__can_reuse_first_leaf(self, *state, result, table_entry)) {
LOG(
"cant_reuse_node symbol:%s, first_leaf_symbol:%s",
SYM_NAME(result->symbol),
TREE_NAME(result),
SYM_NAME(leaf_symbol)
);
reusable_node_advance_past_leaf(&self->reusable_node);
break;
}
LOG("reuse_node symbol:%s", SYM_NAME(result->symbol));
LOG("reuse_node symbol:%s", TREE_NAME(result));
ts_subtree_retain(result);
return result;
}
return NULL;
return NULL_SUBTREE;
}
static bool ts_parser__select_tree(TSParser *self, const Subtree *left, const Subtree *right) {
if (!left) return true;
if (!right) return false;
static bool ts_parser__select_tree(TSParser *self, Subtree left, Subtree right) {
if (!left.ptr) return true;
if (!right.ptr) return false;
if (right->error_cost < left->error_cost) {
LOG("select_smaller_error symbol:%s, over_symbol:%s",
SYM_NAME(right->symbol), SYM_NAME(left->symbol));
if (ts_subtree_error_cost(right) < ts_subtree_error_cost(left)) {
LOG("select_smaller_error symbol:%s, over_symbol:%s", TREE_NAME(right), TREE_NAME(left));
return true;
}
if (left->error_cost < right->error_cost) {
LOG("select_smaller_error symbol:%s, over_symbol:%s",
SYM_NAME(left->symbol), SYM_NAME(right->symbol));
if (ts_subtree_error_cost(left) < ts_subtree_error_cost(right)) {
LOG("select_smaller_error symbol:%s, over_symbol:%s", TREE_NAME(left), TREE_NAME(right));
return false;
}
if (right->dynamic_precedence > left->dynamic_precedence) {
if (ts_subtree_dynamic_precedence(right) > ts_subtree_dynamic_precedence(left)) {
LOG("select_higher_precedence symbol:%s, prec:%u, over_symbol:%s, other_prec:%u",
SYM_NAME(right->symbol), right->dynamic_precedence, SYM_NAME(left->symbol),
left->dynamic_precedence);
TREE_NAME(right), ts_subtree_dynamic_precedence(right), TREE_NAME(left),
ts_subtree_dynamic_precedence(left));
return true;
}
if (left->dynamic_precedence > right->dynamic_precedence) {
if (ts_subtree_dynamic_precedence(left) > ts_subtree_dynamic_precedence(right)) {
LOG("select_higher_precedence symbol:%s, prec:%u, over_symbol:%s, other_prec:%u",
SYM_NAME(left->symbol), left->dynamic_precedence, SYM_NAME(right->symbol),
right->dynamic_precedence);
TREE_NAME(left), ts_subtree_dynamic_precedence(left), TREE_NAME(right),
ts_subtree_dynamic_precedence(right));
return false;
}
if (left->error_cost > 0) return true;
if (ts_subtree_error_cost(left) > 0) return true;
int comparison = ts_subtree_compare(left, right);
switch (comparison) {
case -1:
LOG("select_earlier symbol:%s, over_symbol:%s", SYM_NAME(left->symbol),
SYM_NAME(right->symbol));
LOG("select_earlier symbol:%s, over_symbol:%s", TREE_NAME(left), TREE_NAME(right));
return false;
break;
case 1:
LOG("select_earlier symbol:%s, over_symbol:%s", SYM_NAME(right->symbol),
SYM_NAME(left->symbol));
LOG("select_earlier symbol:%s, over_symbol:%s", TREE_NAME(right), TREE_NAME(left));
return true;
default:
LOG("select_existing symbol:%s, over_symbol:%s", SYM_NAME(left->symbol),
SYM_NAME(right->symbol));
LOG("select_existing symbol:%s, over_symbol:%s", TREE_NAME(left), TREE_NAME(right));
return false;
}
}
static void ts_parser__shift(TSParser *self, StackVersion version, TSStateId state,
const Subtree *lookahead, bool extra) {
const Subtree *subtree_to_push;
if (extra != lookahead->extra) {
Subtree *result = ts_subtree_make_mut(&self->tree_pool, lookahead);
result->extra = extra;
subtree_to_push = result;
Subtree lookahead, bool extra) {
Subtree subtree_to_push;
if (extra != ts_subtree_extra(lookahead)) {
MutableSubtree result = ts_subtree_make_mut(&self->tree_pool, lookahead);
ts_subtree_set_extra(&result);
subtree_to_push = ts_subtree_from_mut(result);
} else {
subtree_to_push = lookahead;
}
bool is_pending = subtree_to_push->child_count > 0;
bool is_pending = ts_subtree_child_count(subtree_to_push) > 0;
ts_stack_push(self->stack, version, subtree_to_push, is_pending, state);
if (subtree_to_push->has_external_tokens) {
if (ts_subtree_has_external_tokens(subtree_to_push)) {
ts_stack_set_last_external_token(
self->stack, version, ts_subtree_last_external_token(subtree_to_push)
);
}
}
static bool ts_parser__replace_children(TSParser *self, Subtree *tree, SubtreeArray *children) {
self->scratch_tree = *tree;
self->scratch_tree.child_count = 0;
ts_subtree_set_children(&self->scratch_tree, children->contents, children->size, self->language);
if (ts_parser__select_tree(self, tree, &self->scratch_tree)) {
*tree = self->scratch_tree;
static bool ts_parser__replace_children(TSParser *self, MutableSubtree *tree, SubtreeArray *children) {
*self->scratch_tree.ptr = *tree->ptr;
self->scratch_tree.ptr->child_count = 0;
ts_subtree_set_children(self->scratch_tree, children->contents, children->size, self->language);
if (ts_parser__select_tree(self, ts_subtree_from_mut(*tree), ts_subtree_from_mut(self->scratch_tree))) {
*tree->ptr = *self->scratch_tree.ptr;
return true;
} else {
return false;
@ -675,11 +674,11 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version, TSSy
// node. They will be re-pushed onto the stack after the parent node is
// created and pushed.
SubtreeArray children = slice.subtrees;
while (children.size > 0 && children.contents[children.size - 1]->extra) {
while (children.size > 0 && ts_subtree_extra(children.contents[children.size - 1])) {
children.size--;
}
Subtree *parent = ts_subtree_new_node(&self->tree_pool,
MutableSubtree parent = ts_subtree_new_node(&self->tree_pool,
symbol, &children, alias_sequence_id, self->language
);
@ -693,11 +692,11 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version, TSSy
i++;
SubtreeArray children = next_slice.subtrees;
while (children.size > 0 && children.contents[children.size - 1]->extra) {
while (children.size > 0 && ts_subtree_extra(children.contents[children.size - 1])) {
children.size--;
}
if (ts_parser__replace_children(self, parent, &children)) {
if (ts_parser__replace_children(self, &parent, &children)) {
ts_subtree_array_delete(&self->tree_pool, &slice.subtrees);
slice = next_slice;
} else {
@ -705,23 +704,23 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version, TSSy
}
}
parent->dynamic_precedence += dynamic_precedence;
parent->alias_sequence_id = alias_sequence_id;
parent.ptr->dynamic_precedence += dynamic_precedence;
parent.ptr->alias_sequence_id = alias_sequence_id;
TSStateId state = ts_stack_state(self->stack, slice_version);
TSStateId next_state = ts_language_next_state(self->language, state, symbol);
if (fragile || pop.size > 1 || initial_version_count > 1) {
parent->fragile_left = true;
parent->fragile_right = true;
parent->parse_state = TS_TREE_STATE_NONE;
parent.ptr->fragile_left = true;
parent.ptr->fragile_right = true;
parent.ptr->parse_state = TS_TREE_STATE_NONE;
} else {
parent->parse_state = state;
parent.ptr->parse_state = state;
}
// Push the parent node onto the stack, along with any extra tokens that
// were previously on top of the stack.
ts_stack_push(self->stack, slice_version, parent, false, next_state);
for (uint32_t j = parent->child_count; j < slice.subtrees.size; j++) {
ts_stack_push(self->stack, slice_version, ts_subtree_from_mut(parent), false, next_state);
for (uint32_t j = parent.ptr->child_count; j < slice.subtrees.size; j++) {
ts_stack_push(self->stack, slice_version, slice.subtrees.contents[j], false, next_state);
}
@ -740,35 +739,40 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version, TSSy
: STACK_VERSION_NONE;
}
static void ts_parser__accept(TSParser *self, StackVersion version, const Subtree *lookahead) {
assert(lookahead->symbol == ts_builtin_sym_end);
static void ts_parser__accept(TSParser *self, StackVersion version, Subtree lookahead) {
assert(ts_subtree_is_eof(lookahead));
ts_stack_push(self->stack, version, lookahead, false, 1);
StackSliceArray pop = ts_stack_pop_all(self->stack, version);
for (uint32_t i = 0; i < pop.size; i++) {
SubtreeArray trees = pop.contents[i].subtrees;
const Subtree *root = NULL;
Subtree root = NULL_SUBTREE;
for (uint32_t j = trees.size - 1; j + 1 > 0; j--) {
const Subtree *child = trees.contents[j];
if (!child->extra) {
for (uint32_t k = 0; k < child->child_count; k++) {
ts_subtree_retain(child->children[k]);
Subtree child = trees.contents[j];
if (!ts_subtree_extra(child)) {
assert(!child.data.is_inline);
uint32_t child_count = ts_subtree_child_count(child);
for (uint32_t k = 0; k < child_count; k++) {
ts_subtree_retain(child.ptr->children[k]);
}
array_splice(&trees, j, 1, child->child_count, child->children);
root = ts_subtree_new_node(
&self->tree_pool, child->symbol, &trees,
child->alias_sequence_id, self->language
);
array_splice(&trees, j, 1, child_count, child.ptr->children);
root = ts_subtree_from_mut(ts_subtree_new_node(
&self->tree_pool,
ts_subtree_symbol(child),
&trees,
child.ptr->alias_sequence_id,
self->language
));
ts_subtree_release(&self->tree_pool, child);
break;
}
}
assert(root && root->ref_count > 0);
assert(root.ptr);
self->accept_count++;
if (self->finished_tree) {
if (self->finished_tree.ptr) {
if (ts_parser__select_tree(self, self->finished_tree, root)) {
ts_subtree_release(&self->tree_pool, self->finished_tree);
self->finished_tree = root;
@ -906,7 +910,7 @@ static void ts_parser__handle_error(TSParser *self, StackVersion version,
Length padding = length_sub(self->lexer.token_end_position, position);
StackVersion version_with_missing_tree = ts_stack_copy_version(self->stack, v);
const Subtree *missing_tree = ts_subtree_new_missing_leaf(
Subtree missing_tree = ts_subtree_new_missing_leaf(
&self->tree_pool, missing_symbol, padding, self->language
);
ts_stack_push(
@ -931,7 +935,7 @@ static void ts_parser__handle_error(TSParser *self, StackVersion version,
}
}
ts_stack_push(self->stack, v, NULL, false, ERROR_STATE);
ts_stack_push(self->stack, v, NULL_SUBTREE, false, ERROR_STATE);
v = (v == version) ? previous_version_count : v + 1;
}
@ -953,15 +957,23 @@ static void ts_parser__halt_parse(TSParser *self) {
ts_stack_position(self->stack, 0)
);
Subtree *filler_node = ts_subtree_new_error(&self->tree_pool, remaining_length, length_zero(), 0, self->language);
filler_node->visible = false;
Subtree filler_node = ts_subtree_new_error(
&self->tree_pool,
0,
length_zero(),
remaining_length,
remaining_length.bytes,
0,
self->language
);
ts_subtree_to_mut_unsafe(filler_node).ptr->visible = false;
ts_stack_push(self->stack, 0, filler_node, false, 0);
SubtreeArray children = array_new();
Subtree *root_error = ts_subtree_new_error_node(&self->tree_pool, &children, self->language);
Subtree root_error = ts_subtree_new_error_node(&self->tree_pool, &children, false, self->language);
ts_stack_push(self->stack, 0, root_error, false, 0);
Subtree *eof = ts_subtree_new_leaf(
Subtree eof = ts_subtree_new_leaf(
&self->tree_pool,
ts_builtin_sym_end,
length_zero(),
@ -999,10 +1011,13 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, un
SubtreeArray error_trees = ts_stack_pop_error(self->stack, slice.version);
if (error_trees.size > 0) {
assert(error_trees.size == 1);
const Subtree *error_tree = error_trees.contents[0];
array_splice(&slice.subtrees, 0, 0, error_tree->child_count, error_tree->children);
for (unsigned j = 0; j < error_tree->child_count; j++) {
ts_subtree_retain(slice.subtrees.contents[j]);
Subtree error_tree = error_trees.contents[0];
uint32_t error_child_count = ts_subtree_child_count(error_tree);
if (error_child_count > 0) {
array_splice(&slice.subtrees, 0, 0, error_child_count, error_tree.ptr->children);
for (unsigned j = 0; j < error_child_count; j++) {
ts_subtree_retain(slice.subtrees.contents[j]);
}
}
ts_subtree_array_delete(&self->tree_pool, &error_trees);
}
@ -1010,15 +1025,14 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, un
SubtreeArray trailing_extras = ts_subtree_array_remove_trailing_extras(&slice.subtrees);
if (slice.subtrees.size > 0) {
Subtree *error = ts_subtree_new_error_node(&self->tree_pool, &slice.subtrees, self->language);
error->extra = true;
Subtree error = ts_subtree_new_error_node(&self->tree_pool, &slice.subtrees, true, self->language);
ts_stack_push(self->stack, slice.version, error, false, goal_state);
} else {
array_delete(&slice.subtrees);
}
for (unsigned j = 0; j < trailing_extras.size; j++) {
const Subtree *tree = trailing_extras.contents[j];
Subtree tree = trailing_extras.contents[j];
ts_stack_push(self->stack, slice.version, tree, false, goal_state);
}
@ -1029,7 +1043,7 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, un
return previous_version != STACK_VERSION_NONE;
}
static void ts_parser__recover(TSParser *self, StackVersion version, const Subtree *lookahead) {
static void ts_parser__recover(TSParser *self, StackVersion version, Subtree lookahead) {
bool did_recover = false;
unsigned previous_version_count = ts_stack_version_count(self->stack);
Length position = ts_stack_position(self->stack, version);
@ -1037,7 +1051,7 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr
unsigned node_count_since_error = ts_stack_node_count_since_error(self->stack, version);
unsigned current_error_cost = ts_stack_error_cost(self->stack, version);
if (summary && lookahead->symbol != ts_builtin_sym_error) {
if (summary && !ts_subtree_is_error(lookahead)) {
for (unsigned i = 0; i < summary->size; i++) {
StackSummaryEntry entry = summary->contents[i];
@ -1066,7 +1080,7 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr
(position.extent.row - entry.position.extent.row) * ERROR_COST_PER_SKIPPED_LINE;
if (ts_parser__better_version_exists(self, version, false, new_cost)) break;
if (ts_language_has_actions(self->language, entry.state, lookahead->symbol)) {
if (ts_language_has_actions(self->language, entry.state, ts_subtree_symbol(lookahead))) {
if (ts_parser__recover_to_state(self, version, depth, entry.state)) {
did_recover = true;
LOG("recover_to_previous state:%u, depth:%u", entry.state, depth);
@ -1089,10 +1103,10 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr
return;
}
if (lookahead->symbol == ts_builtin_sym_end) {
if (ts_subtree_is_eof(lookahead)) {
LOG("recover_eof");
SubtreeArray children = array_new();
const Subtree *parent = ts_subtree_new_error_node(&self->tree_pool, &children, self->language);
Subtree parent = ts_subtree_new_error_node(&self->tree_pool, &children, false, self->language);
ts_stack_push(self->stack, version, parent, false, 1);
ts_parser__accept(self, version, lookahead);
return;
@ -1110,18 +1124,18 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr
}
unsigned n;
const TSParseAction *actions = ts_language_actions(self->language, 1, lookahead->symbol, &n);
const TSParseAction *actions = ts_language_actions(self->language, 1, ts_subtree_symbol(lookahead), &n);
if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.extra) {
Subtree *mutable_lookahead = ts_subtree_make_mut(&self->tree_pool, lookahead);
mutable_lookahead->extra = true;
lookahead = mutable_lookahead;
MutableSubtree mutable_lookahead = ts_subtree_make_mut(&self->tree_pool, lookahead);
ts_subtree_set_extra(&mutable_lookahead);
lookahead = ts_subtree_from_mut(mutable_lookahead);
}
LOG("skip_token symbol:%s", SYM_NAME(lookahead->symbol));
LOG("skip_token symbol:%s", TREE_NAME(lookahead));
SubtreeArray children = array_new();
array_reserve(&children, 1);
array_push(&children, lookahead);
const Subtree *error_repeat = ts_subtree_new_node(
MutableSubtree error_repeat = ts_subtree_new_node(
&self->tree_pool,
ts_builtin_sym_error_repeat,
&children,
@ -1134,7 +1148,7 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr
assert(pop.size == 1);
assert(pop.contents[0].subtrees.size == 1);
ts_stack_renumber_version(self->stack, pop.contents[0].version, version);
array_push(&pop.contents[0].subtrees, error_repeat);
array_push(&pop.contents[0].subtrees, ts_subtree_from_mut(error_repeat));
error_repeat = ts_subtree_new_node(
&self->tree_pool,
ts_builtin_sym_error_repeat,
@ -1144,9 +1158,9 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr
);
}
ts_stack_push(self->stack, version, error_repeat, false, ERROR_STATE);
ts_stack_push(self->stack, version, ts_subtree_from_mut(error_repeat), false, ERROR_STATE);
if (lookahead->has_external_tokens) {
if (ts_subtree_has_external_tokens(lookahead)) {
ts_stack_set_last_external_token(
self->stack, version, ts_subtree_last_external_token(lookahead)
);
@ -1156,11 +1170,11 @@ static void ts_parser__recover(TSParser *self, StackVersion version, const Subtr
static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_node_reuse) {
TSStateId state = ts_stack_state(self->stack, version);
uint32_t position = ts_stack_position(self->stack, version).bytes;
const Subtree *last_external_token = ts_stack_last_external_token(self->stack, version);
Subtree last_external_token = ts_stack_last_external_token(self->stack, version);
bool did_reuse = true;
const Subtree *lookahead = NULL;
TableEntry table_entry;
Subtree lookahead = NULL_SUBTREE;
TableEntry table_entry = {.action_count = 0};
// If possible, reuse a node from the previous syntax tree.
if (allow_node_reuse) {
@ -1170,7 +1184,7 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
}
// Otherwise, try to reuse the token previously returned by the lexer.
if (!lookahead) {
if (!lookahead.ptr) {
did_reuse = false;
lookahead = ts_parser__get_cached_token(
self, state, position, last_external_token, &table_entry
@ -1178,10 +1192,10 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
}
// Otherwise, re-run the lexer.
if (!lookahead) {
if (!lookahead.ptr) {
lookahead = ts_parser__lex(self, version, state);
ts_parser__set_cached_token(self, position, last_external_token, lookahead);
ts_language_table_entry(self->language, state, lookahead->symbol, &table_entry);
ts_language_table_entry(self->language, state, ts_subtree_symbol(lookahead), &table_entry);
}
for (;;) {
@ -1206,9 +1220,9 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
LOG("shift state:%u", next_state);
}
if (lookahead->child_count > 0) {
if (ts_subtree_child_count(lookahead) > 0) {
ts_parser__breakdown_lookahead(self, &lookahead, state, &self->reusable_node);
next_state = ts_language_next_state(self->language, state, lookahead->symbol);
next_state = ts_language_next_state(self->language, state, ts_subtree_symbol(lookahead));
}
ts_parser__shift(self, version, next_state, lookahead, action.params.extra);
@ -1237,7 +1251,7 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
}
case TSParseActionTypeRecover: {
if (lookahead->child_count > 0) {
if (ts_subtree_child_count(lookahead) > 0) {
ts_parser__breakdown_lookahead(self, &lookahead, ERROR_STATE, &self->reusable_node);
}
@ -1261,18 +1275,21 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
continue;
}
if (lookahead->is_keyword && lookahead->symbol != self->language->keyword_capture_token) {
if (
ts_subtree_is_keyword(lookahead) &&
ts_subtree_symbol(lookahead) != self->language->keyword_capture_token
) {
ts_language_table_entry(self->language, state, self->language->keyword_capture_token, &table_entry);
if (table_entry.action_count > 0) {
LOG(
"switch from_keyword:%s, to_word_token:%s",
SYM_NAME(lookahead->symbol),
TREE_NAME(lookahead),
SYM_NAME(self->language->keyword_capture_token)
);
Subtree *mutable_lookahead = ts_subtree_make_mut(&self->tree_pool, lookahead);
mutable_lookahead->symbol = self->language->keyword_capture_token;
lookahead = mutable_lookahead;
MutableSubtree mutable_lookahead = ts_subtree_make_mut(&self->tree_pool, lookahead);
ts_subtree_set_symbol(&mutable_lookahead, self->language->keyword_capture_token);
lookahead = ts_subtree_from_mut(mutable_lookahead);
continue;
}
}
@ -1395,14 +1412,15 @@ TSParser *ts_parser_new() {
array_reserve(&self->reduce_actions, 4);
self->tree_pool = ts_subtree_pool_new(32);
self->stack = ts_stack_new(&self->tree_pool);
self->finished_tree = NULL;
self->finished_tree = NULL_SUBTREE;
self->reusable_node = reusable_node_new();
self->dot_graph_file = NULL;
self->halt_on_error = false;
self->enabled = true;
self->operation_limit = SIZE_MAX;
self->old_tree = NULL;
ts_parser__set_cached_token(self, 0, NULL, NULL);
self->old_tree = NULL_SUBTREE;
self->scratch_tree.ptr = &self->scratch_tree_data;
ts_parser__set_cached_token(self, 0, NULL_SUBTREE, NULL_SUBTREE);
return self;
}
@ -1411,12 +1429,12 @@ void ts_parser_delete(TSParser *self) {
if (self->reduce_actions.contents) {
array_delete(&self->reduce_actions);
}
if (self->old_tree) {
if (self->old_tree.ptr) {
ts_subtree_release(&self->tree_pool, self->old_tree);
self->old_tree = NULL;
self->old_tree = NULL_SUBTREE;
}
ts_lexer_delete(&self->lexer);
ts_parser__set_cached_token(self, 0, NULL, NULL);
ts_parser__set_cached_token(self, 0, NULL_SUBTREE, NULL_SUBTREE);
ts_subtree_pool_delete(&self->tree_pool);
reusable_node_delete(&self->reusable_node);
ts_parser_set_language(self, NULL);
@ -1489,18 +1507,18 @@ void ts_parser_reset(TSParser *self) {
self->language->external_scanner.deserialize(self->external_scanner_payload, NULL, 0);
}
if (self->old_tree) {
if (self->old_tree.ptr) {
ts_subtree_release(&self->tree_pool, self->old_tree);
self->old_tree = NULL;
self->old_tree = NULL_SUBTREE;
}
reusable_node_clear(&self->reusable_node);
ts_lexer_reset(&self->lexer, length_zero());
ts_stack_clear(self->stack);
ts_parser__set_cached_token(self, 0, NULL, NULL);
if (self->finished_tree) {
ts_parser__set_cached_token(self, 0, NULL_SUBTREE, NULL_SUBTREE);
if (self->finished_tree.ptr) {
ts_subtree_release(&self->tree_pool, self->finished_tree);
self->finished_tree = NULL;
self->finished_tree = NULL_SUBTREE;
}
self->accept_count = 0;
}
@ -1552,7 +1570,7 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) {
}
unsigned min_error_cost = ts_parser__condense_stack(self);
if (self->finished_tree && self->finished_tree->error_cost < min_error_cost) {
if (self->finished_tree.ptr && ts_subtree_error_cost(self->finished_tree) < min_error_cost) {
break;
} else if (self->halt_on_error && min_error_cost > 0) {
ts_parser__halt_parse(self);
@ -1565,7 +1583,7 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) {
LOG_TREE();
TSTree *result = ts_tree_new(self->finished_tree, self->language);
self->finished_tree = NULL;
self->finished_tree = NULL_SUBTREE;
ts_parser_reset(self);
return result;
}